| blob | 8949e4b782f63079aee5ca90f7a46e3e6fb43563 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
11 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
12 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
13 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
15 define amdgpu_kernel void @flat_workgroup_unordered_load(
16 ; GFX7-LABEL: flat_workgroup_unordered_load:
17 ; GFX7: ; %bb.0: ; %entry
18 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
19 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
20 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
21 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
22 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
23 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
24 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
25 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
26 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
27 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
28 ; GFX7-NEXT: flat_store_dword v[0:1], v2
29 ; GFX7-NEXT: s_endpgm
30 ;
31 ; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
32 ; GFX10-WGP: ; %bb.0: ; %entry
33 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
34 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
35 ; GFX10-WGP-NEXT: s_nop 0
36 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
37 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
38 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
39 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
40 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
41 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
42 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
43 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
44 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
45 ; GFX10-WGP-NEXT: s_endpgm
46 ;
47 ; GFX10-CU-LABEL: flat_workgroup_unordered_load:
48 ; GFX10-CU: ; %bb.0: ; %entry
49 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
50 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
51 ; GFX10-CU-NEXT: s_nop 0
52 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
53 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
54 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
55 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
56 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
57 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
58 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
59 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
60 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
61 ; GFX10-CU-NEXT: s_endpgm
62 ;
63 ; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
64 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
65 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
66 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
67 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
68 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
69 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
70 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
71 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
72 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
73 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
74 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
75 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
76 ; SKIP-CACHE-INV-NEXT: s_endpgm
77 ;
78 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
79 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
80 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
81 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
82 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
83 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
84 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
85 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
86 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
87 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
88 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
89 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
90 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
91 ;
92 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_load:
93 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
94 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
95 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
96 ; GFX90A-TGSPLIT-NEXT: s_nop 0
97 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
98 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
99 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
100 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
101 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
102 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
103 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
104 ; GFX90A-TGSPLIT-NEXT: s_endpgm
105 ;
106 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_load:
107 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
108 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
109 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
110 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
111 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
112 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
113 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
114 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
115 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
116 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
117 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
118 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
119 ;
120 ; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_load:
121 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
122 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
123 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
124 ; GFX940-TGSPLIT-NEXT: s_nop 0
125 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
126 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
127 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
128 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
129 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
130 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
131 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
132 ; GFX940-TGSPLIT-NEXT: s_endpgm
133 ;
134 ; GFX11-WGP-LABEL: flat_workgroup_unordered_load:
135 ; GFX11-WGP: ; %bb.0: ; %entry
136 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
137 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
138 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
139 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
140 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
141 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
142 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
143 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
144 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
145 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
146 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
147 ; GFX11-WGP-NEXT: s_endpgm
148 ;
149 ; GFX11-CU-LABEL: flat_workgroup_unordered_load:
150 ; GFX11-CU: ; %bb.0: ; %entry
151 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
152 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
153 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
154 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
155 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
156 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
157 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
158 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
159 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
160 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
161 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
162 ; GFX11-CU-NEXT: s_endpgm
163 ;
164 ; GFX12-WGP-LABEL: flat_workgroup_unordered_load:
165 ; GFX12-WGP: ; %bb.0: ; %entry
166 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
167 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
168 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
169 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
170 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
171 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
172 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
173 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
174 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
175 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
176 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
177 ; GFX12-WGP-NEXT: s_endpgm
178 ;
179 ; GFX12-CU-LABEL: flat_workgroup_unordered_load:
180 ; GFX12-CU: ; %bb.0: ; %entry
181 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
182 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
183 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
184 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
185 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
186 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
187 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
188 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
189 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
190 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
191 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
192 ; GFX12-CU-NEXT: s_endpgm
193 ptr %in, ptr %out) {
194 entry:
195 %val = load atomic i32, ptr %in syncscope("workgroup") unordered, align 4
196 store i32 %val, ptr %out
197 ret void
198 }
200 define amdgpu_kernel void @flat_workgroup_monotonic_load(
201 ; GFX7-LABEL: flat_workgroup_monotonic_load:
202 ; GFX7: ; %bb.0: ; %entry
203 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
204 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
205 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
206 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
207 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
208 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
209 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
210 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
211 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
212 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
213 ; GFX7-NEXT: flat_store_dword v[0:1], v2
214 ; GFX7-NEXT: s_endpgm
215 ;
216 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
217 ; GFX10-WGP: ; %bb.0: ; %entry
218 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
219 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
220 ; GFX10-WGP-NEXT: s_nop 0
221 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
222 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
223 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
224 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
225 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
226 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
227 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
228 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
229 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
230 ; GFX10-WGP-NEXT: s_endpgm
231 ;
232 ; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
233 ; GFX10-CU: ; %bb.0: ; %entry
234 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
235 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
236 ; GFX10-CU-NEXT: s_nop 0
237 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
238 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
239 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
240 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
241 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
242 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
243 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
244 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
245 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
246 ; GFX10-CU-NEXT: s_endpgm
247 ;
248 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
249 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
250 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
251 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
252 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
253 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
254 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
255 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
256 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
257 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
258 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
259 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
260 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
261 ; SKIP-CACHE-INV-NEXT: s_endpgm
262 ;
263 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
264 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
265 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
266 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
267 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
268 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
269 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
270 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
271 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
272 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
273 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
274 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
275 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
276 ;
277 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
278 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
279 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
280 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
281 ; GFX90A-TGSPLIT-NEXT: s_nop 0
282 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
283 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
284 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
285 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
286 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
287 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
288 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
289 ; GFX90A-TGSPLIT-NEXT: s_endpgm
290 ;
291 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_load:
292 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
293 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
294 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
295 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
296 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
297 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
298 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
299 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
300 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
301 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
302 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
303 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
304 ;
305 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_load:
306 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
307 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
308 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
309 ; GFX940-TGSPLIT-NEXT: s_nop 0
310 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
311 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
312 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
313 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
314 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
315 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
316 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
317 ; GFX940-TGSPLIT-NEXT: s_endpgm
318 ;
319 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_load:
320 ; GFX11-WGP: ; %bb.0: ; %entry
321 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
322 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
323 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
324 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
325 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
326 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
327 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
328 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
329 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
330 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
331 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
332 ; GFX11-WGP-NEXT: s_endpgm
333 ;
334 ; GFX11-CU-LABEL: flat_workgroup_monotonic_load:
335 ; GFX11-CU: ; %bb.0: ; %entry
336 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
337 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
338 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
339 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
340 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
341 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
342 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
343 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
344 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
345 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
346 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
347 ; GFX11-CU-NEXT: s_endpgm
348 ;
349 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_load:
350 ; GFX12-WGP: ; %bb.0: ; %entry
351 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
352 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
353 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
354 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
355 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
356 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
357 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
358 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
359 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
360 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
361 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
362 ; GFX12-WGP-NEXT: s_endpgm
363 ;
364 ; GFX12-CU-LABEL: flat_workgroup_monotonic_load:
365 ; GFX12-CU: ; %bb.0: ; %entry
366 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
367 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
368 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
369 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
370 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
371 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
372 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
373 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
374 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
375 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
376 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
377 ; GFX12-CU-NEXT: s_endpgm
378 ptr %in, ptr %out) {
379 entry:
380 %val = load atomic i32, ptr %in syncscope("workgroup") monotonic, align 4
381 store i32 %val, ptr %out
382 ret void
383 }
385 define amdgpu_kernel void @flat_workgroup_acquire_load(
386 ; GFX7-LABEL: flat_workgroup_acquire_load:
387 ; GFX7: ; %bb.0: ; %entry
388 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
389 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
390 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
391 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
392 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
393 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
394 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
395 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
396 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
397 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
398 ; GFX7-NEXT: s_waitcnt vmcnt(0)
399 ; GFX7-NEXT: flat_store_dword v[0:1], v2
400 ; GFX7-NEXT: s_endpgm
401 ;
402 ; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
403 ; GFX10-WGP: ; %bb.0: ; %entry
404 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
405 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
406 ; GFX10-WGP-NEXT: s_nop 0
407 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
408 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
409 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
410 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
411 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
412 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
413 ; GFX10-WGP-NEXT: buffer_gl0_inv
414 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
415 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
416 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
417 ; GFX10-WGP-NEXT: s_endpgm
418 ;
419 ; GFX10-CU-LABEL: flat_workgroup_acquire_load:
420 ; GFX10-CU: ; %bb.0: ; %entry
421 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
422 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
423 ; GFX10-CU-NEXT: s_nop 0
424 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
425 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
426 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
427 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
428 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
429 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
430 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
431 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
432 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
433 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
434 ; GFX10-CU-NEXT: s_endpgm
435 ;
436 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
437 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
438 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
439 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
440 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
441 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
442 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
443 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
444 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
445 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
446 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
447 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
448 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
449 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
450 ; SKIP-CACHE-INV-NEXT: s_endpgm
451 ;
452 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
453 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
454 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
455 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
456 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
457 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
458 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
459 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
460 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
461 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
462 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
463 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
464 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
465 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
466 ;
467 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_load:
468 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
469 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
470 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
471 ; GFX90A-TGSPLIT-NEXT: s_nop 0
472 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
473 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
474 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
475 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
476 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
477 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
478 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
479 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
480 ; GFX90A-TGSPLIT-NEXT: s_endpgm
481 ;
482 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_load:
483 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
484 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
485 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
486 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
487 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
488 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
489 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
490 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
491 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
492 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
493 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
494 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
495 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
496 ;
497 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_load:
498 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
499 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
500 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
501 ; GFX940-TGSPLIT-NEXT: s_nop 0
502 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
503 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
504 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
505 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
506 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
507 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
508 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
509 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
510 ; GFX940-TGSPLIT-NEXT: s_endpgm
511 ;
512 ; GFX11-WGP-LABEL: flat_workgroup_acquire_load:
513 ; GFX11-WGP: ; %bb.0: ; %entry
514 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
515 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
516 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
517 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
518 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
519 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
520 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
521 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
522 ; GFX11-WGP-NEXT: buffer_gl0_inv
523 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
524 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
525 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
526 ; GFX11-WGP-NEXT: s_endpgm
527 ;
528 ; GFX11-CU-LABEL: flat_workgroup_acquire_load:
529 ; GFX11-CU: ; %bb.0: ; %entry
530 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
531 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
532 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
533 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
534 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
535 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
536 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
537 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
538 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
539 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
540 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
541 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
542 ; GFX11-CU-NEXT: s_endpgm
543 ;
544 ; GFX12-WGP-LABEL: flat_workgroup_acquire_load:
545 ; GFX12-WGP: ; %bb.0: ; %entry
546 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
547 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
548 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
549 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
550 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
551 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
552 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
553 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
554 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
555 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
556 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
557 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
558 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
559 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
560 ; GFX12-WGP-NEXT: s_endpgm
561 ;
562 ; GFX12-CU-LABEL: flat_workgroup_acquire_load:
563 ; GFX12-CU: ; %bb.0: ; %entry
564 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
565 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
566 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
567 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
568 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
569 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
570 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
571 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
572 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
573 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
574 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
575 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
576 ; GFX12-CU-NEXT: s_endpgm
577 ptr %in, ptr %out) {
578 entry:
579 %val = load atomic i32, ptr %in syncscope("workgroup") acquire, align 4
580 store i32 %val, ptr %out
581 ret void
582 }
584 define amdgpu_kernel void @flat_workgroup_seq_cst_load(
585 ; GFX7-LABEL: flat_workgroup_seq_cst_load:
586 ; GFX7: ; %bb.0: ; %entry
587 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
588 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
589 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
590 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
591 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
592 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
593 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
594 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
595 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
596 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
597 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
598 ; GFX7-NEXT: s_waitcnt vmcnt(0)
599 ; GFX7-NEXT: flat_store_dword v[0:1], v2
600 ; GFX7-NEXT: s_endpgm
601 ;
602 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
603 ; GFX10-WGP: ; %bb.0: ; %entry
604 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
605 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
606 ; GFX10-WGP-NEXT: s_nop 0
607 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
608 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
609 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
610 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
611 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
612 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
613 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
614 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
615 ; GFX10-WGP-NEXT: buffer_gl0_inv
616 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
617 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
618 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
619 ; GFX10-WGP-NEXT: s_endpgm
620 ;
621 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
622 ; GFX10-CU: ; %bb.0: ; %entry
623 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
624 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
625 ; GFX10-CU-NEXT: s_nop 0
626 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
627 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
628 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
629 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
630 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
631 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
632 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
633 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
634 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
635 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
636 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
637 ; GFX10-CU-NEXT: s_endpgm
638 ;
639 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
640 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
641 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
642 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
643 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
644 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
645 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
646 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
647 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
648 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
649 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
650 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
651 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
652 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
653 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
654 ; SKIP-CACHE-INV-NEXT: s_endpgm
655 ;
656 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
657 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
658 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
659 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
660 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
661 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
662 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
663 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
664 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
665 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
666 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
667 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
668 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
669 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
670 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
671 ;
672 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
673 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
674 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
675 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
676 ; GFX90A-TGSPLIT-NEXT: s_nop 0
677 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
678 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
679 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
680 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
681 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
682 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
683 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
684 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
685 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
686 ; GFX90A-TGSPLIT-NEXT: s_endpgm
687 ;
688 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_load:
689 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
690 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
691 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
692 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
693 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
694 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
695 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
696 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
697 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
698 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
699 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
700 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
701 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
702 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
703 ;
704 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_load:
705 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
706 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
707 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
708 ; GFX940-TGSPLIT-NEXT: s_nop 0
709 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
710 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
711 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
712 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
713 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
714 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
715 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
716 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
717 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
718 ; GFX940-TGSPLIT-NEXT: s_endpgm
719 ;
720 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_load:
721 ; GFX11-WGP: ; %bb.0: ; %entry
722 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
723 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
724 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
725 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
726 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
727 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
728 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
729 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
730 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
731 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
732 ; GFX11-WGP-NEXT: buffer_gl0_inv
733 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
734 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
735 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
736 ; GFX11-WGP-NEXT: s_endpgm
737 ;
738 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_load:
739 ; GFX11-CU: ; %bb.0: ; %entry
740 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
741 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
742 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
743 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
744 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
745 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
746 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
747 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
748 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
749 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
750 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
751 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
752 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
753 ; GFX11-CU-NEXT: s_endpgm
754 ;
755 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_load:
756 ; GFX12-WGP: ; %bb.0: ; %entry
757 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
758 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
759 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
760 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
761 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
762 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
763 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
764 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
765 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
766 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
767 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
768 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
769 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
770 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
771 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
772 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
773 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
774 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
775 ; GFX12-WGP-NEXT: s_endpgm
776 ;
777 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_load:
778 ; GFX12-CU: ; %bb.0: ; %entry
779 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
780 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
781 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
782 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
783 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
784 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
785 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
786 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
787 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
788 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
789 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
790 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
791 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
792 ; GFX12-CU-NEXT: s_endpgm
793 ptr %in, ptr %out) {
794 entry:
795 %val = load atomic i32, ptr %in syncscope("workgroup") seq_cst, align 4
796 store i32 %val, ptr %out
797 ret void
798 }
800 define amdgpu_kernel void @flat_workgroup_unordered_store(
801 ; GFX7-LABEL: flat_workgroup_unordered_store:
802 ; GFX7: ; %bb.0: ; %entry
803 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
804 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
805 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
806 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
807 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
808 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
809 ; GFX7-NEXT: flat_store_dword v[0:1], v2
810 ; GFX7-NEXT: s_endpgm
811 ;
812 ; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
813 ; GFX10-WGP: ; %bb.0: ; %entry
814 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
815 ; GFX10-WGP-NEXT: s_nop 0
816 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
817 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
818 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
819 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
820 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
821 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
822 ; GFX10-WGP-NEXT: s_endpgm
823 ;
824 ; GFX10-CU-LABEL: flat_workgroup_unordered_store:
825 ; GFX10-CU: ; %bb.0: ; %entry
826 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
827 ; GFX10-CU-NEXT: s_nop 0
828 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
829 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
830 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
831 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
832 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
833 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
834 ; GFX10-CU-NEXT: s_endpgm
835 ;
836 ; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
837 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
838 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
839 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
840 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
841 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
842 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
843 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
844 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
845 ; SKIP-CACHE-INV-NEXT: s_endpgm
846 ;
847 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
848 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
849 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
850 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
851 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
852 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
853 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
854 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
855 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
856 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
857 ;
858 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_unordered_store:
859 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
860 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
861 ; GFX90A-TGSPLIT-NEXT: s_nop 0
862 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
863 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
864 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
865 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
866 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
867 ; GFX90A-TGSPLIT-NEXT: s_endpgm
868 ;
869 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_unordered_store:
870 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
871 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
872 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
873 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
874 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
875 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
876 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
877 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
878 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
879 ;
880 ; GFX940-TGSPLIT-LABEL: flat_workgroup_unordered_store:
881 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
882 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
883 ; GFX940-TGSPLIT-NEXT: s_nop 0
884 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
885 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
886 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
887 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
888 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
889 ; GFX940-TGSPLIT-NEXT: s_endpgm
890 ;
891 ; GFX11-WGP-LABEL: flat_workgroup_unordered_store:
892 ; GFX11-WGP: ; %bb.0: ; %entry
893 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
894 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
895 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
896 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
897 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
898 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
899 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
900 ; GFX11-WGP-NEXT: s_endpgm
901 ;
902 ; GFX11-CU-LABEL: flat_workgroup_unordered_store:
903 ; GFX11-CU: ; %bb.0: ; %entry
904 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
905 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
906 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
907 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
908 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
909 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
910 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
911 ; GFX11-CU-NEXT: s_endpgm
912 ;
913 ; GFX12-WGP-LABEL: flat_workgroup_unordered_store:
914 ; GFX12-WGP: ; %bb.0: ; %entry
915 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
916 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
917 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
918 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
919 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
920 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
921 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
922 ; GFX12-WGP-NEXT: s_endpgm
923 ;
924 ; GFX12-CU-LABEL: flat_workgroup_unordered_store:
925 ; GFX12-CU: ; %bb.0: ; %entry
926 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
927 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
928 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
929 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
930 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
931 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
932 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
933 ; GFX12-CU-NEXT: s_endpgm
934 i32 %in, ptr %out) {
935 entry:
936 store atomic i32 %in, ptr %out syncscope("workgroup") unordered, align 4
937 ret void
938 }
940 define amdgpu_kernel void @flat_workgroup_monotonic_store(
941 ; GFX7-LABEL: flat_workgroup_monotonic_store:
942 ; GFX7: ; %bb.0: ; %entry
943 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
944 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
945 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
946 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
947 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
948 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
949 ; GFX7-NEXT: flat_store_dword v[0:1], v2
950 ; GFX7-NEXT: s_endpgm
951 ;
952 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
953 ; GFX10-WGP: ; %bb.0: ; %entry
954 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
955 ; GFX10-WGP-NEXT: s_nop 0
956 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
957 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
958 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
959 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
960 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
961 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
962 ; GFX10-WGP-NEXT: s_endpgm
963 ;
964 ; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
965 ; GFX10-CU: ; %bb.0: ; %entry
966 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
967 ; GFX10-CU-NEXT: s_nop 0
968 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
969 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
970 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
971 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
972 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
973 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
974 ; GFX10-CU-NEXT: s_endpgm
975 ;
976 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
977 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
978 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
979 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
980 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
981 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
982 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
983 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
984 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
985 ; SKIP-CACHE-INV-NEXT: s_endpgm
986 ;
987 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
988 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
989 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
990 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
991 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
992 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
993 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
994 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
995 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
996 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
997 ;
998 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
999 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1000 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
1001 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1002 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1003 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1004 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1005 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1006 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1007 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1008 ;
1009 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_store:
1010 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1011 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1012 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1013 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1014 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1015 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1016 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1017 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1018 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1019 ;
1020 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_store:
1021 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1022 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1023 ; GFX940-TGSPLIT-NEXT: s_nop 0
1024 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1025 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1026 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1027 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1028 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1029 ; GFX940-TGSPLIT-NEXT: s_endpgm
1030 ;
1031 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_store:
1032 ; GFX11-WGP: ; %bb.0: ; %entry
1033 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1034 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1035 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1036 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1037 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1038 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1039 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1040 ; GFX11-WGP-NEXT: s_endpgm
1041 ;
1042 ; GFX11-CU-LABEL: flat_workgroup_monotonic_store:
1043 ; GFX11-CU: ; %bb.0: ; %entry
1044 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1045 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1046 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1047 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1048 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1049 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1050 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1051 ; GFX11-CU-NEXT: s_endpgm
1052 ;
1053 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_store:
1054 ; GFX12-WGP: ; %bb.0: ; %entry
1055 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1056 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1057 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1058 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1059 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1060 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1061 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
1062 ; GFX12-WGP-NEXT: s_endpgm
1063 ;
1064 ; GFX12-CU-LABEL: flat_workgroup_monotonic_store:
1065 ; GFX12-CU: ; %bb.0: ; %entry
1066 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1067 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1068 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1069 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1070 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1071 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1072 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
1073 ; GFX12-CU-NEXT: s_endpgm
1074 i32 %in, ptr %out) {
1075 entry:
1076 store atomic i32 %in, ptr %out syncscope("workgroup") monotonic, align 4
1077 ret void
1078 }
1080 define amdgpu_kernel void @flat_workgroup_release_store(
1081 ; GFX7-LABEL: flat_workgroup_release_store:
1082 ; GFX7: ; %bb.0: ; %entry
1083 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
1084 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
1085 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1086 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1087 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1088 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1089 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1090 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1091 ; GFX7-NEXT: s_endpgm
1092 ;
1093 ; GFX10-WGP-LABEL: flat_workgroup_release_store:
1094 ; GFX10-WGP: ; %bb.0: ; %entry
1095 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
1096 ; GFX10-WGP-NEXT: s_nop 0
1097 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1098 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1099 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1100 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1101 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1102 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1103 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1104 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1105 ; GFX10-WGP-NEXT: s_endpgm
1106 ;
1107 ; GFX10-CU-LABEL: flat_workgroup_release_store:
1108 ; GFX10-CU: ; %bb.0: ; %entry
1109 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
1110 ; GFX10-CU-NEXT: s_nop 0
1111 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1112 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1113 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1114 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1115 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1116 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1117 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1118 ; GFX10-CU-NEXT: s_endpgm
1119 ;
1120 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
1121 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1122 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
1123 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
1124 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1125 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1126 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1127 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1128 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1129 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1130 ; SKIP-CACHE-INV-NEXT: s_endpgm
1131 ;
1132 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
1133 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1134 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
1135 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1136 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1137 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1138 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1139 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1140 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1141 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1142 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1143 ;
1144 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_store:
1145 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1146 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
1147 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1148 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1149 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1150 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1151 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1152 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1153 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1154 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1155 ;
1156 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_store:
1157 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1158 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1159 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1160 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1161 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1162 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1164 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1165 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1166 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1167 ;
1168 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_store:
1169 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1170 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1171 ; GFX940-TGSPLIT-NEXT: s_nop 0
1172 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1173 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1174 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1175 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1176 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1177 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1178 ; GFX940-TGSPLIT-NEXT: s_endpgm
1179 ;
1180 ; GFX11-WGP-LABEL: flat_workgroup_release_store:
1181 ; GFX11-WGP: ; %bb.0: ; %entry
1182 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1183 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1184 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1185 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1186 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1187 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1188 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1189 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1190 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1191 ; GFX11-WGP-NEXT: s_endpgm
1192 ;
1193 ; GFX11-CU-LABEL: flat_workgroup_release_store:
1194 ; GFX11-CU: ; %bb.0: ; %entry
1195 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1196 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1197 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1198 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1199 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1200 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1201 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1202 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1203 ; GFX11-CU-NEXT: s_endpgm
1204 ;
1205 ; GFX12-WGP-LABEL: flat_workgroup_release_store:
1206 ; GFX12-WGP: ; %bb.0: ; %entry
1207 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1208 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1209 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1210 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1211 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1212 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1213 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
1214 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1215 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1216 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1217 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1218 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
1219 ; GFX12-WGP-NEXT: s_endpgm
1220 ;
1221 ; GFX12-CU-LABEL: flat_workgroup_release_store:
1222 ; GFX12-CU: ; %bb.0: ; %entry
1223 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1224 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1225 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1226 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1227 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1228 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1229 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1230 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
1231 ; GFX12-CU-NEXT: s_endpgm
1232 i32 %in, ptr %out) {
1233 entry:
1234 store atomic i32 %in, ptr %out syncscope("workgroup") release, align 4
1235 ret void
1236 }
1238 define amdgpu_kernel void @flat_workgroup_seq_cst_store(
1239 ; GFX7-LABEL: flat_workgroup_seq_cst_store:
1240 ; GFX7: ; %bb.0: ; %entry
1241 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
1242 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
1243 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1244 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1245 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1246 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1247 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1248 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1249 ; GFX7-NEXT: s_endpgm
1250 ;
1251 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
1252 ; GFX10-WGP: ; %bb.0: ; %entry
1253 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
1254 ; GFX10-WGP-NEXT: s_nop 0
1255 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1256 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1257 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1258 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1259 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1260 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1261 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1262 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1263 ; GFX10-WGP-NEXT: s_endpgm
1264 ;
1265 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
1266 ; GFX10-CU: ; %bb.0: ; %entry
1267 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
1268 ; GFX10-CU-NEXT: s_nop 0
1269 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1270 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1271 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1272 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1273 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1274 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1275 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1276 ; GFX10-CU-NEXT: s_endpgm
1277 ;
1278 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
1279 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1280 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
1281 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
1282 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1283 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1284 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1285 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1286 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1287 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1288 ; SKIP-CACHE-INV-NEXT: s_endpgm
1289 ;
1290 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1291 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1292 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
1293 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1294 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1295 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1296 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1297 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1298 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1299 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1300 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1301 ;
1302 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1303 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1304 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
1305 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1306 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
1307 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1308 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1309 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1310 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1311 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1312 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1313 ;
1314 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1315 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1316 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1317 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1318 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1319 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1320 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1321 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1322 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1323 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1324 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1325 ;
1326 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_store:
1327 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1328 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
1329 ; GFX940-TGSPLIT-NEXT: s_nop 0
1330 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
1331 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1332 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1333 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1334 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1335 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1336 ; GFX940-TGSPLIT-NEXT: s_endpgm
1337 ;
1338 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_store:
1339 ; GFX11-WGP: ; %bb.0: ; %entry
1340 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1341 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1342 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1343 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1344 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1345 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1346 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1347 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1348 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1349 ; GFX11-WGP-NEXT: s_endpgm
1350 ;
1351 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_store:
1352 ; GFX11-CU: ; %bb.0: ; %entry
1353 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1354 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1355 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1356 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1357 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1358 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1359 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1360 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1361 ; GFX11-CU-NEXT: s_endpgm
1362 ;
1363 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_store:
1364 ; GFX12-WGP: ; %bb.0: ; %entry
1365 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
1366 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1367 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1368 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1369 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1370 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1371 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
1372 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1373 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1374 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1375 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1376 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
1377 ; GFX12-WGP-NEXT: s_endpgm
1378 ;
1379 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_store:
1380 ; GFX12-CU: ; %bb.0: ; %entry
1381 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
1382 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
1383 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1384 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1385 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1386 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1387 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1388 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
1389 ; GFX12-CU-NEXT: s_endpgm
1390 i32 %in, ptr %out) {
1391 entry:
1392 store atomic i32 %in, ptr %out syncscope("workgroup") seq_cst, align 4
1393 ret void
1394 }
1396 define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
1397 ; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
1398 ; GFX7: ; %bb.0: ; %entry
1399 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1400 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1401 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1402 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1403 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1404 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1405 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1406 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1407 ; GFX7-NEXT: s_endpgm
1408 ;
1409 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1410 ; GFX10-WGP: ; %bb.0: ; %entry
1411 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1412 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1413 ; GFX10-WGP-NEXT: s_nop 0
1414 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
1415 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1416 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1417 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1418 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1419 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1420 ; GFX10-WGP-NEXT: s_endpgm
1421 ;
1422 ; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1423 ; GFX10-CU: ; %bb.0: ; %entry
1424 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1425 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1426 ; GFX10-CU-NEXT: s_nop 0
1427 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
1428 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1429 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1430 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1431 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1432 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1433 ; GFX10-CU-NEXT: s_endpgm
1434 ;
1435 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
1436 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1437 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
1438 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1439 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1440 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1441 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1442 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1443 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1444 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1445 ; SKIP-CACHE-INV-NEXT: s_endpgm
1446 ;
1447 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1448 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1449 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1450 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1451 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1452 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1453 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1454 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1455 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1456 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1457 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1458 ;
1459 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1460 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1461 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1462 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1463 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1464 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1465 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1466 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1467 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1468 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1469 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1470 ;
1471 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1472 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1473 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1474 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1475 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1476 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1477 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1478 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1479 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1480 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1481 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1482 ;
1483 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_atomicrmw:
1484 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1485 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1486 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1487 ; GFX940-TGSPLIT-NEXT: s_nop 0
1488 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1489 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1490 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1491 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1492 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1493 ; GFX940-TGSPLIT-NEXT: s_endpgm
1494 ;
1495 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1496 ; GFX11-WGP: ; %bb.0: ; %entry
1497 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1498 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1499 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1500 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1501 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1502 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1503 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1504 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1505 ; GFX11-WGP-NEXT: s_endpgm
1506 ;
1507 ; GFX11-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1508 ; GFX11-CU: ; %bb.0: ; %entry
1509 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1510 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1511 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1512 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1513 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1514 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1515 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1516 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1517 ; GFX11-CU-NEXT: s_endpgm
1518 ;
1519 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
1520 ; GFX12-WGP: ; %bb.0: ; %entry
1521 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1522 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1523 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1524 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1525 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1526 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1527 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1528 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
1529 ; GFX12-WGP-NEXT: s_endpgm
1530 ;
1531 ; GFX12-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
1532 ; GFX12-CU: ; %bb.0: ; %entry
1533 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1534 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1535 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1536 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1537 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1538 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1539 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1540 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1541 ; GFX12-CU-NEXT: s_endpgm
1542 ptr %out, i32 %in) {
1543 entry:
1544 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") monotonic
1545 ret void
1546 }
1548 define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
1549 ; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
1550 ; GFX7: ; %bb.0: ; %entry
1551 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1552 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1553 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1554 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1555 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1556 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1557 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1558 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1559 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1560 ; GFX7-NEXT: s_endpgm
1561 ;
1562 ; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1563 ; GFX10-WGP: ; %bb.0: ; %entry
1564 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1565 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1566 ; GFX10-WGP-NEXT: s_nop 0
1567 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
1568 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1569 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1570 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1571 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1572 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1573 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1574 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1575 ; GFX10-WGP-NEXT: buffer_gl0_inv
1576 ; GFX10-WGP-NEXT: s_endpgm
1577 ;
1578 ; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1579 ; GFX10-CU: ; %bb.0: ; %entry
1580 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1581 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1582 ; GFX10-CU-NEXT: s_nop 0
1583 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
1584 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1585 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1586 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1587 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1588 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1589 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1590 ; GFX10-CU-NEXT: s_endpgm
1591 ;
1592 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
1593 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1594 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
1595 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1596 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1597 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1598 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1599 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1600 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1601 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1602 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1603 ; SKIP-CACHE-INV-NEXT: s_endpgm
1604 ;
1605 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1606 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1607 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1608 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1609 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1610 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1611 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1612 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1613 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1614 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1615 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1616 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1617 ;
1618 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1619 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1620 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1621 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1622 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1623 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1624 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1625 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1626 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1627 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1628 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1629 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
1630 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1631 ;
1632 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1633 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1634 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1635 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1636 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1637 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1638 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1639 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1640 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1641 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1642 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1643 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1644 ;
1645 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_atomicrmw:
1646 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1647 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1648 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1649 ; GFX940-TGSPLIT-NEXT: s_nop 0
1650 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1651 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1652 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1653 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1654 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1655 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1656 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
1657 ; GFX940-TGSPLIT-NEXT: s_endpgm
1658 ;
1659 ; GFX11-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1660 ; GFX11-WGP: ; %bb.0: ; %entry
1661 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1662 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1663 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1664 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1665 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1666 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1667 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1668 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1669 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1670 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1671 ; GFX11-WGP-NEXT: buffer_gl0_inv
1672 ; GFX11-WGP-NEXT: s_endpgm
1673 ;
1674 ; GFX11-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1675 ; GFX11-CU: ; %bb.0: ; %entry
1676 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1677 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1678 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1679 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1680 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1681 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1682 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1683 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1684 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1685 ; GFX11-CU-NEXT: s_endpgm
1686 ;
1687 ; GFX12-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
1688 ; GFX12-WGP: ; %bb.0: ; %entry
1689 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1690 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1691 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1692 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1693 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1694 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1695 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1696 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
1697 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
1698 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
1699 ; GFX12-WGP-NEXT: s_endpgm
1700 ;
1701 ; GFX12-CU-LABEL: flat_workgroup_acquire_atomicrmw:
1702 ; GFX12-CU: ; %bb.0: ; %entry
1703 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1704 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1705 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1706 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1707 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1708 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1709 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1710 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1711 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1712 ; GFX12-CU-NEXT: s_endpgm
1713 ptr %out, i32 %in) {
1714 entry:
1715 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
1716 ret void
1717 }
1719 define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
1720 ; GFX7-LABEL: flat_workgroup_release_atomicrmw:
1721 ; GFX7: ; %bb.0: ; %entry
1722 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1723 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1724 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1725 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1726 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1727 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1728 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1729 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1730 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1731 ; GFX7-NEXT: s_endpgm
1732 ;
1733 ; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
1734 ; GFX10-WGP: ; %bb.0: ; %entry
1735 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1736 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1737 ; GFX10-WGP-NEXT: s_nop 0
1738 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
1739 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1740 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1741 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1742 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1743 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1744 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1745 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1746 ; GFX10-WGP-NEXT: s_endpgm
1747 ;
1748 ; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
1749 ; GFX10-CU: ; %bb.0: ; %entry
1750 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1751 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1752 ; GFX10-CU-NEXT: s_nop 0
1753 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
1754 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1755 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1756 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1757 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1758 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1759 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1760 ; GFX10-CU-NEXT: s_endpgm
1761 ;
1762 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
1763 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1764 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
1765 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1766 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1767 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1768 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1769 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1770 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1771 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1772 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1773 ; SKIP-CACHE-INV-NEXT: s_endpgm
1774 ;
1775 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1776 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1777 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1778 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1779 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1780 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1781 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1782 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1783 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1784 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1785 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1786 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1787 ;
1788 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1789 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1790 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1791 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1792 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1793 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1794 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1795 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1796 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1797 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1798 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1799 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1800 ;
1801 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1802 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1803 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1804 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1805 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1806 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1807 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1808 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1809 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1810 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1811 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1812 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1813 ;
1814 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_atomicrmw:
1815 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1816 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1817 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1818 ; GFX940-TGSPLIT-NEXT: s_nop 0
1819 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1820 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1821 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1822 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1823 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1824 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1825 ; GFX940-TGSPLIT-NEXT: s_endpgm
1826 ;
1827 ; GFX11-WGP-LABEL: flat_workgroup_release_atomicrmw:
1828 ; GFX11-WGP: ; %bb.0: ; %entry
1829 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1830 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1831 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1832 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1833 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
1834 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
1835 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1836 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1837 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1838 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1839 ; GFX11-WGP-NEXT: s_endpgm
1840 ;
1841 ; GFX11-CU-LABEL: flat_workgroup_release_atomicrmw:
1842 ; GFX11-CU: ; %bb.0: ; %entry
1843 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1844 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1845 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1846 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1847 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1848 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1849 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1850 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1851 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1852 ; GFX11-CU-NEXT: s_endpgm
1853 ;
1854 ; GFX12-WGP-LABEL: flat_workgroup_release_atomicrmw:
1855 ; GFX12-WGP: ; %bb.0: ; %entry
1856 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1857 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1858 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1859 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1860 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1861 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1862 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1863 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
1864 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1865 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1866 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1867 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1868 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
1869 ; GFX12-WGP-NEXT: s_endpgm
1870 ;
1871 ; GFX12-CU-LABEL: flat_workgroup_release_atomicrmw:
1872 ; GFX12-CU: ; %bb.0: ; %entry
1873 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1874 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1875 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1876 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1877 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1878 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1879 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1880 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1881 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1882 ; GFX12-CU-NEXT: s_endpgm
1883 ptr %out, i32 %in) {
1884 entry:
1885 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") release
1886 ret void
1887 }
1889 define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
1890 ; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
1891 ; GFX7: ; %bb.0: ; %entry
1892 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1893 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1894 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1895 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1896 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1897 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1898 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1899 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1900 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1901 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1902 ; GFX7-NEXT: s_endpgm
1903 ;
1904 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
1905 ; GFX10-WGP: ; %bb.0: ; %entry
1906 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1907 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1908 ; GFX10-WGP-NEXT: s_nop 0
1909 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
1910 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1911 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
1912 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
1913 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
1914 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1915 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1916 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1917 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1918 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1919 ; GFX10-WGP-NEXT: buffer_gl0_inv
1920 ; GFX10-WGP-NEXT: s_endpgm
1921 ;
1922 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
1923 ; GFX10-CU: ; %bb.0: ; %entry
1924 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1925 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1926 ; GFX10-CU-NEXT: s_nop 0
1927 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
1928 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1929 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
1930 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
1931 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
1932 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1933 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1934 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1935 ; GFX10-CU-NEXT: s_endpgm
1936 ;
1937 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
1938 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1939 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
1940 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1941 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1942 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1943 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1944 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1945 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1946 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1947 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1948 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1949 ; SKIP-CACHE-INV-NEXT: s_endpgm
1950 ;
1951 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1952 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1953 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1954 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1955 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1956 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1957 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1958 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1959 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1960 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1961 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1962 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1963 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1964 ;
1965 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1966 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1967 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1968 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1969 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1970 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
1971 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1972 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
1973 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1974 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1975 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1976 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1977 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
1978 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1979 ;
1980 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1981 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1982 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1983 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1984 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1985 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
1986 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1987 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1988 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
1989 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1990 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1991 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1992 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1993 ;
1994 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_atomicrmw:
1995 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1996 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1997 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1998 ; GFX940-TGSPLIT-NEXT: s_nop 0
1999 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
2000 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2001 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
2002 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
2003 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2004 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2005 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2006 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2007 ; GFX940-TGSPLIT-NEXT: s_endpgm
2008 ;
2009 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
2010 ; GFX11-WGP: ; %bb.0: ; %entry
2011 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
2012 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2013 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
2014 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2015 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
2016 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
2017 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
2018 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2019 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2020 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
2021 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2022 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2023 ; GFX11-WGP-NEXT: buffer_gl0_inv
2024 ; GFX11-WGP-NEXT: s_endpgm
2025 ;
2026 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
2027 ; GFX11-CU: ; %bb.0: ; %entry
2028 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
2029 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2030 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
2031 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2032 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
2033 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
2034 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
2035 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2036 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
2037 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2038 ; GFX11-CU-NEXT: s_endpgm
2039 ;
2040 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
2041 ; GFX12-WGP: ; %bb.0: ; %entry
2042 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
2043 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2044 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
2045 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2046 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
2047 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
2048 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
2049 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2050 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2051 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2052 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2053 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2054 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
2055 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
2056 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2057 ; GFX12-WGP-NEXT: s_endpgm
2058 ;
2059 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
2060 ; GFX12-CU: ; %bb.0: ; %entry
2061 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
2062 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2063 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
2064 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2065 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
2066 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
2067 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
2068 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2069 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
2070 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2071 ; GFX12-CU-NEXT: s_endpgm
2072 ptr %out, i32 %in) {
2073 entry:
2074 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
2075 ret void
2076 }
2078 define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
2079 ; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
2080 ; GFX7: ; %bb.0: ; %entry
2081 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
2082 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2083 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
2084 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2085 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2086 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2087 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2088 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2089 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
2090 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2091 ; GFX7-NEXT: s_endpgm
2092 ;
2093 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
2094 ; GFX10-WGP: ; %bb.0: ; %entry
2095 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
2096 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2097 ; GFX10-WGP-NEXT: s_nop 0
2098 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
2099 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2100 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
2101 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
2102 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
2103 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2104 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2105 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
2106 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2107 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2108 ; GFX10-WGP-NEXT: buffer_gl0_inv
2109 ; GFX10-WGP-NEXT: s_endpgm
2110 ;
2111 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
2112 ; GFX10-CU: ; %bb.0: ; %entry
2113 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
2114 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2115 ; GFX10-CU-NEXT: s_nop 0
2116 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
2117 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2118 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
2119 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
2120 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
2121 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2122 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
2123 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2124 ; GFX10-CU-NEXT: s_endpgm
2125 ;
2126 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
2127 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2128 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
2129 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
2130 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
2131 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2132 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
2133 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
2134 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
2135 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2136 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
2137 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2138 ; SKIP-CACHE-INV-NEXT: s_endpgm
2139 ;
2140 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
2141 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2142 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
2143 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2144 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2145 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
2146 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2147 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
2148 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
2149 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2150 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2151 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2152 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2153 ;
2154 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
2155 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2156 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
2157 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2158 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2159 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
2160 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2161 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
2162 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
2163 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2164 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2165 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2166 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2167 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2168 ;
2169 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
2170 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2171 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
2172 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
2173 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2174 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
2175 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2176 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
2177 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
2178 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2179 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2180 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2181 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2182 ;
2183 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_atomicrmw:
2184 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2185 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
2186 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
2187 ; GFX940-TGSPLIT-NEXT: s_nop 0
2188 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
2189 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2190 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
2191 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
2192 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2193 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
2194 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2195 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2196 ; GFX940-TGSPLIT-NEXT: s_endpgm
2197 ;
2198 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
2199 ; GFX11-WGP: ; %bb.0: ; %entry
2200 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
2201 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2202 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
2203 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2204 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
2205 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
2206 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
2207 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2208 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2209 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
2210 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2211 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2212 ; GFX11-WGP-NEXT: buffer_gl0_inv
2213 ; GFX11-WGP-NEXT: s_endpgm
2214 ;
2215 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
2216 ; GFX11-CU: ; %bb.0: ; %entry
2217 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
2218 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2219 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
2220 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2221 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
2222 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
2223 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
2224 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2225 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
2226 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2227 ; GFX11-CU-NEXT: s_endpgm
2228 ;
2229 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
2230 ; GFX12-WGP: ; %bb.0: ; %entry
2231 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
2232 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2233 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
2234 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2235 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
2236 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
2237 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
2238 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2239 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2240 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2241 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2242 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2243 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
2244 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
2245 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2246 ; GFX12-WGP-NEXT: s_endpgm
2247 ;
2248 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
2249 ; GFX12-CU: ; %bb.0: ; %entry
2250 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
2251 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
2252 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
2253 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2254 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
2255 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
2256 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
2257 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2258 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
2259 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2260 ; GFX12-CU-NEXT: s_endpgm
2261 ptr %out, i32 %in) {
2262 entry:
2263 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
2264 ret void
2265 }
2267 define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
2268 ; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2269 ; GFX7: ; %bb.0: ; %entry
2270 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2271 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2272 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2273 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2274 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2275 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2276 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2277 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2278 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2279 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2280 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2281 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2282 ; GFX7-NEXT: s_endpgm
2283 ;
2284 ; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2285 ; GFX10-WGP: ; %bb.0: ; %entry
2286 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2287 ; GFX10-WGP-NEXT: s_nop 0
2288 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2289 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2290 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2291 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2292 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2293 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2294 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2295 ; GFX10-WGP-NEXT: buffer_gl0_inv
2296 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2297 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2298 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2299 ; GFX10-WGP-NEXT: s_endpgm
2300 ;
2301 ; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2302 ; GFX10-CU: ; %bb.0: ; %entry
2303 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2304 ; GFX10-CU-NEXT: s_nop 0
2305 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2306 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2307 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2308 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2309 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2310 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2311 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2312 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2313 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2314 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2315 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2316 ; GFX10-CU-NEXT: s_endpgm
2317 ;
2318 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2319 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2320 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2321 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
2322 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2323 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2324 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2325 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2326 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2327 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2328 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2329 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2330 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2331 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2332 ; SKIP-CACHE-INV-NEXT: s_endpgm
2333 ;
2334 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2335 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2336 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2337 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2338 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2339 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2340 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2341 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2342 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2343 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2344 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2345 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2346 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2347 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2348 ;
2349 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2350 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2351 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2352 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2353 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2354 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2355 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2356 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2357 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2358 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2359 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2360 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2361 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2362 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2363 ;
2364 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2365 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2366 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2367 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2368 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2369 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2370 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2371 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2372 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2373 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2374 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2375 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2376 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2377 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2378 ;
2379 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2380 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2381 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2382 ; GFX940-TGSPLIT-NEXT: s_nop 0
2383 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2384 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2385 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2386 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2387 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2388 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2389 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2390 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2391 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2392 ; GFX940-TGSPLIT-NEXT: s_endpgm
2393 ;
2394 ; GFX11-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2395 ; GFX11-WGP: ; %bb.0: ; %entry
2396 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2397 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2398 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2399 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2401 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2402 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2403 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2404 ; GFX11-WGP-NEXT: buffer_gl0_inv
2405 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2406 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2407 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2408 ; GFX11-WGP-NEXT: s_endpgm
2409 ;
2410 ; GFX11-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2411 ; GFX11-CU: ; %bb.0: ; %entry
2412 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2413 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2414 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2415 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2416 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2417 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2418 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2419 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2420 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2421 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2422 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2423 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2424 ; GFX11-CU-NEXT: s_endpgm
2425 ;
2426 ; GFX12-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2427 ; GFX12-WGP: ; %bb.0: ; %entry
2428 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2429 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2430 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2431 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2432 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2433 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2434 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
2435 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2436 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2437 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2438 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2439 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2440 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2441 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2442 ; GFX12-WGP-NEXT: s_endpgm
2443 ;
2444 ; GFX12-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
2445 ; GFX12-CU: ; %bb.0: ; %entry
2446 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2447 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2448 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2449 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2450 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2451 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2452 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
2453 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2454 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2455 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2456 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
2457 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2458 ; GFX12-CU-NEXT: s_endpgm
2459 ptr %out, i32 %in) {
2460 entry:
2461 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
2462 store i32 %val, ptr %out, align 4
2463 ret void
2464 }
2466 define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
2467 ; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2468 ; GFX7: ; %bb.0: ; %entry
2469 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2470 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2471 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2472 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2473 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2474 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2475 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2476 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2477 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2478 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2479 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2480 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2481 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2482 ; GFX7-NEXT: s_endpgm
2483 ;
2484 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2485 ; GFX10-WGP: ; %bb.0: ; %entry
2486 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2487 ; GFX10-WGP-NEXT: s_nop 0
2488 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2489 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2490 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2491 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2492 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2493 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2494 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2495 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2496 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2497 ; GFX10-WGP-NEXT: buffer_gl0_inv
2498 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2499 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2500 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2501 ; GFX10-WGP-NEXT: s_endpgm
2502 ;
2503 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2504 ; GFX10-CU: ; %bb.0: ; %entry
2505 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2506 ; GFX10-CU-NEXT: s_nop 0
2507 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2508 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2509 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2510 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2511 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2512 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2513 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2514 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2515 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2516 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2517 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2518 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2519 ; GFX10-CU-NEXT: s_endpgm
2520 ;
2521 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2522 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2523 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2524 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
2525 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2526 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2527 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2528 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2529 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2530 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2531 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2532 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2533 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2534 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2535 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2536 ; SKIP-CACHE-INV-NEXT: s_endpgm
2537 ;
2538 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2539 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2540 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2541 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2542 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2543 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2544 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2545 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2546 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2547 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2548 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2549 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2550 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2551 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2552 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2553 ;
2554 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2555 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2556 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2557 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2558 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2559 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2560 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2561 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2562 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2563 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2564 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2565 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2566 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2567 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2568 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2569 ;
2570 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2571 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2572 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2573 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2574 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2575 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2576 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2577 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2578 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2579 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2580 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2581 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2582 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2583 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2584 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2585 ;
2586 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2587 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2588 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2589 ; GFX940-TGSPLIT-NEXT: s_nop 0
2590 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2591 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2592 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2593 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2594 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2595 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2596 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2597 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2598 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2599 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2600 ; GFX940-TGSPLIT-NEXT: s_endpgm
2601 ;
2602 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2603 ; GFX11-WGP: ; %bb.0: ; %entry
2604 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2605 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2606 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2607 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2608 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2609 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2610 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2611 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2612 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2613 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2614 ; GFX11-WGP-NEXT: buffer_gl0_inv
2615 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2616 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2617 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2618 ; GFX11-WGP-NEXT: s_endpgm
2619 ;
2620 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2621 ; GFX11-CU: ; %bb.0: ; %entry
2622 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2623 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2624 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2625 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2626 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2627 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2628 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2629 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2630 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2631 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2632 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2633 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2634 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2635 ; GFX11-CU-NEXT: s_endpgm
2636 ;
2637 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2638 ; GFX12-WGP: ; %bb.0: ; %entry
2639 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2640 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2641 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2642 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2643 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2644 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2645 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2646 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2647 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2648 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2649 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2650 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
2651 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2652 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2653 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2654 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2655 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2656 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2657 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2658 ; GFX12-WGP-NEXT: s_endpgm
2659 ;
2660 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
2661 ; GFX12-CU: ; %bb.0: ; %entry
2662 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2663 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2664 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2665 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2666 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2667 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2668 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2669 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
2670 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2671 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2672 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2673 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
2674 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2675 ; GFX12-CU-NEXT: s_endpgm
2676 ptr %out, i32 %in) {
2677 entry:
2678 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
2679 store i32 %val, ptr %out, align 4
2680 ret void
2681 }
2683 define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
2684 ; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2685 ; GFX7: ; %bb.0: ; %entry
2686 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2687 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2688 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2689 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2690 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2691 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2692 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2693 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2694 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2695 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2696 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2697 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2698 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2699 ; GFX7-NEXT: s_endpgm
2700 ;
2701 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2702 ; GFX10-WGP: ; %bb.0: ; %entry
2703 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2704 ; GFX10-WGP-NEXT: s_nop 0
2705 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2706 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2707 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2708 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2709 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
2710 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2711 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2712 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2713 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2714 ; GFX10-WGP-NEXT: buffer_gl0_inv
2715 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2716 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2717 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
2718 ; GFX10-WGP-NEXT: s_endpgm
2719 ;
2720 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2721 ; GFX10-CU: ; %bb.0: ; %entry
2722 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2723 ; GFX10-CU-NEXT: s_nop 0
2724 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2725 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2726 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2727 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2728 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
2729 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2730 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2731 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2732 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2733 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2734 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2735 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
2736 ; GFX10-CU-NEXT: s_endpgm
2737 ;
2738 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2739 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2740 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2741 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
2742 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2743 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2744 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2745 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2746 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2747 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2748 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2749 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2750 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2751 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2752 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
2753 ; SKIP-CACHE-INV-NEXT: s_endpgm
2754 ;
2755 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2756 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2757 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2758 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2759 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2760 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2761 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2762 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2763 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2764 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2765 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2766 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2767 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2768 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
2769 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2770 ;
2771 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2772 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2773 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2774 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2775 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2776 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2777 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2778 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
2779 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2780 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2781 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2782 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2783 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
2784 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
2785 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2786 ;
2787 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2788 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2789 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2790 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2791 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2792 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2793 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2794 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2795 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2796 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2797 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2798 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2799 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2800 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2801 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2802 ;
2803 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2804 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2805 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2806 ; GFX940-TGSPLIT-NEXT: s_nop 0
2807 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2808 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2809 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2810 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
2811 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2812 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
2813 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2814 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2815 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2816 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
2817 ; GFX940-TGSPLIT-NEXT: s_endpgm
2818 ;
2819 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2820 ; GFX11-WGP: ; %bb.0: ; %entry
2821 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2822 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2823 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2824 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2825 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2826 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
2827 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2828 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2829 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2830 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2831 ; GFX11-WGP-NEXT: buffer_gl0_inv
2832 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
2833 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
2834 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
2835 ; GFX11-WGP-NEXT: s_endpgm
2836 ;
2837 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2838 ; GFX11-CU: ; %bb.0: ; %entry
2839 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2840 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2841 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2842 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2843 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2844 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
2845 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2846 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
2847 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2848 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
2849 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
2850 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2851 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
2852 ; GFX11-CU-NEXT: s_endpgm
2853 ;
2854 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2855 ; GFX12-WGP: ; %bb.0: ; %entry
2856 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2857 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2858 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2859 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2860 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2861 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
2862 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2863 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2864 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2865 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2866 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2867 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
2868 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2869 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2870 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2871 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2872 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
2873 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
2874 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
2875 ; GFX12-WGP-NEXT: s_endpgm
2876 ;
2877 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
2878 ; GFX12-CU: ; %bb.0: ; %entry
2879 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2880 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2881 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2882 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2883 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2884 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
2885 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2886 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
2887 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2888 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
2889 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
2890 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
2891 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
2892 ; GFX12-CU-NEXT: s_endpgm
2893 ptr %out, i32 %in) {
2894 entry:
2895 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
2896 store i32 %val, ptr %out, align 4
2897 ret void
2898 }
2900 define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
2901 ; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2902 ; GFX7: ; %bb.0: ; %entry
2903 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
2904 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2905 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
2906 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
2907 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
2908 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2909 ; GFX7-NEXT: s_mov_b32 s4, s8
2910 ; GFX7-NEXT: s_mov_b32 s5, s9
2911 ; GFX7-NEXT: s_mov_b32 s9, s10
2912 ; GFX7-NEXT: s_mov_b32 s8, s11
2913 ; GFX7-NEXT: s_add_u32 s4, s4, s9
2914 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
2915 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2916 ; GFX7-NEXT: s_mov_b32 s5, s8
2917 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
2918 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2919 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2920 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
2921 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2922 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2923 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2924 ; GFX7-NEXT: s_endpgm
2925 ;
2926 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2927 ; GFX10-WGP: ; %bb.0: ; %entry
2928 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
2929 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2930 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
2931 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
2932 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
2933 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2934 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
2935 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
2936 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
2937 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
2938 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
2939 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
2940 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2941 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
2942 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
2943 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
2944 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2945 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
2946 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
2947 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
2948 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2949 ; GFX10-WGP-NEXT: s_endpgm
2950 ;
2951 ; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2952 ; GFX10-CU: ; %bb.0: ; %entry
2953 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
2954 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
2955 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
2956 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
2957 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
2958 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2959 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
2960 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
2961 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
2962 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
2963 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
2964 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
2965 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
2966 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
2967 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
2968 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
2969 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2970 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
2971 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
2972 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
2973 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2974 ; GFX10-CU-NEXT: s_endpgm
2975 ;
2976 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
2977 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2978 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
2979 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
2980 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
2981 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
2982 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
2983 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2984 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
2985 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
2986 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
2987 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
2988 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
2989 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
2990 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
2991 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
2992 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
2993 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
2994 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
2995 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
2996 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2997 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2998 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2999 ; SKIP-CACHE-INV-NEXT: s_endpgm
3000 ;
3001 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3002 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3003 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3004 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3005 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3006 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3007 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3008 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3009 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3010 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3011 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3012 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3013 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3014 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3015 ;
3016 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3017 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3018 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3019 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3020 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3021 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3022 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3023 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3024 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3025 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3026 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3027 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3028 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3029 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3030 ;
3031 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3032 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3033 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3034 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3035 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3036 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3037 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3038 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3039 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3040 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3041 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3042 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3043 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3044 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3045 ;
3046 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3047 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3048 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3049 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3050 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3051 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3052 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3053 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3054 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3055 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3056 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3057 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3058 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3059 ; GFX940-TGSPLIT-NEXT: s_endpgm
3060 ;
3061 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3062 ; GFX11-WGP: ; %bb.0: ; %entry
3063 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3064 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3065 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3066 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3067 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3068 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3069 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3070 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3071 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3072 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3073 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3074 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3075 ; GFX11-WGP-NEXT: s_endpgm
3076 ;
3077 ; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3078 ; GFX11-CU: ; %bb.0: ; %entry
3079 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3080 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3081 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3082 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3083 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3084 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3085 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3086 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3087 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3088 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3089 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3090 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3091 ; GFX11-CU-NEXT: s_endpgm
3092 ;
3093 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3094 ; GFX12-WGP: ; %bb.0: ; %entry
3095 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3096 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3097 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3098 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3099 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3100 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3101 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3102 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3103 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3104 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3105 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3106 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
3107 ; GFX12-WGP-NEXT: s_endpgm
3108 ;
3109 ; GFX12-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
3110 ; GFX12-CU: ; %bb.0: ; %entry
3111 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3112 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3113 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3114 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3115 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3116 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3117 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3118 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3119 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3120 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3121 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3122 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3123 ; GFX12-CU-NEXT: s_endpgm
3124 ptr %out, i32 %in, i32 %old) {
3125 entry:
3126 %gep = getelementptr i32, ptr %out, i32 4
3127 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
3128 ret void
3129 }
3131 define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
3132 ; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3133 ; GFX7: ; %bb.0: ; %entry
3134 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3135 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3136 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3137 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3138 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3139 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3140 ; GFX7-NEXT: s_mov_b32 s4, s8
3141 ; GFX7-NEXT: s_mov_b32 s5, s9
3142 ; GFX7-NEXT: s_mov_b32 s9, s10
3143 ; GFX7-NEXT: s_mov_b32 s8, s11
3144 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3145 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3146 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3147 ; GFX7-NEXT: s_mov_b32 s5, s8
3148 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3149 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3150 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3151 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3152 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3153 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3154 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3155 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3156 ; GFX7-NEXT: s_endpgm
3157 ;
3158 ; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3159 ; GFX10-WGP: ; %bb.0: ; %entry
3160 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
3161 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3162 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3163 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3164 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3165 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3166 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3167 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3168 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3169 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3170 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3171 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3172 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3173 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3174 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3175 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3176 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3177 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3178 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3179 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3180 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3181 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3182 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3183 ; GFX10-WGP-NEXT: buffer_gl0_inv
3184 ; GFX10-WGP-NEXT: s_endpgm
3185 ;
3186 ; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3187 ; GFX10-CU: ; %bb.0: ; %entry
3188 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
3189 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3190 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3191 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3192 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3193 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3194 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3195 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3196 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3197 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3198 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3199 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3200 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3201 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3202 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3203 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3204 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3205 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3206 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3207 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3208 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3209 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3210 ; GFX10-CU-NEXT: s_endpgm
3211 ;
3212 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3213 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3214 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
3215 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3216 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3217 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3218 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3219 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3220 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3221 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3222 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3223 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3224 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3225 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3226 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3227 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3228 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3229 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3230 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3231 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3232 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3233 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3234 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3235 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3236 ; SKIP-CACHE-INV-NEXT: s_endpgm
3237 ;
3238 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3239 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3240 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3241 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3242 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3243 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3244 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3245 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3246 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3247 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3248 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3249 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3250 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3251 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3252 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3253 ;
3254 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3255 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3256 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3257 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3258 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3259 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3260 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3261 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3262 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3263 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3264 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3265 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3266 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3267 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3268 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3269 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3270 ;
3271 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3272 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3273 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3274 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3275 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3276 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3277 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3278 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3279 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3280 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3281 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3282 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3283 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3284 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3285 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3286 ;
3287 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3288 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3289 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3290 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3291 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3292 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3293 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3294 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3295 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3296 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3297 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3298 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3299 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3300 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3301 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
3302 ; GFX940-TGSPLIT-NEXT: s_endpgm
3303 ;
3304 ; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3305 ; GFX11-WGP: ; %bb.0: ; %entry
3306 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3307 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3308 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3309 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3310 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3311 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3312 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3313 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3314 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3315 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3316 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3317 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3318 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3319 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3320 ; GFX11-WGP-NEXT: buffer_gl0_inv
3321 ; GFX11-WGP-NEXT: s_endpgm
3322 ;
3323 ; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3324 ; GFX11-CU: ; %bb.0: ; %entry
3325 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3326 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3327 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3328 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3329 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3330 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3331 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3332 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3333 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3334 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3335 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3336 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3337 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3338 ; GFX11-CU-NEXT: s_endpgm
3339 ;
3340 ; GFX12-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3341 ; GFX12-WGP: ; %bb.0: ; %entry
3342 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3343 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3344 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3345 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3346 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3347 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3348 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3349 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3350 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3351 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3352 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3353 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
3354 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
3355 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
3356 ; GFX12-WGP-NEXT: s_endpgm
3357 ;
3358 ; GFX12-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
3359 ; GFX12-CU: ; %bb.0: ; %entry
3360 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3361 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3362 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3363 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3364 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3365 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3366 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3367 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3368 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3369 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3370 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3371 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3372 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3373 ; GFX12-CU-NEXT: s_endpgm
3374 ptr %out, i32 %in, i32 %old) {
3375 entry:
3376 %gep = getelementptr i32, ptr %out, i32 4
3377 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
3378 ret void
3379 }
3381 define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
3382 ; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3383 ; GFX7: ; %bb.0: ; %entry
3384 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3385 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3386 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3387 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3388 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3389 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3390 ; GFX7-NEXT: s_mov_b32 s4, s8
3391 ; GFX7-NEXT: s_mov_b32 s5, s9
3392 ; GFX7-NEXT: s_mov_b32 s9, s10
3393 ; GFX7-NEXT: s_mov_b32 s8, s11
3394 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3395 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3396 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3397 ; GFX7-NEXT: s_mov_b32 s5, s8
3398 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3399 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3400 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3401 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3402 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3403 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3404 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3405 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3406 ; GFX7-NEXT: s_endpgm
3407 ;
3408 ; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3409 ; GFX10-WGP: ; %bb.0: ; %entry
3410 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
3411 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3412 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3413 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3414 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3415 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3416 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3417 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3418 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3419 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3420 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3421 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3422 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3423 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3424 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3425 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3426 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3427 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3428 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3429 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3430 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3431 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3432 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3433 ; GFX10-WGP-NEXT: s_endpgm
3434 ;
3435 ; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3436 ; GFX10-CU: ; %bb.0: ; %entry
3437 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
3438 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3439 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3440 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3441 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3442 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3443 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3444 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3445 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3446 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3447 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3448 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3449 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3450 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3451 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3452 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3453 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3454 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3455 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3456 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3457 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3458 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3459 ; GFX10-CU-NEXT: s_endpgm
3460 ;
3461 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3462 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3463 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
3464 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3465 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3466 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3467 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3468 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3469 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3470 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3471 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3472 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3473 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3474 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3475 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3476 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3477 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3478 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3479 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3480 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3481 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3482 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3483 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3484 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3485 ; SKIP-CACHE-INV-NEXT: s_endpgm
3486 ;
3487 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3488 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3489 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3490 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3491 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3492 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3493 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3494 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3495 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3496 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3497 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3498 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3499 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3500 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3501 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3502 ;
3503 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3504 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3505 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3506 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3507 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3508 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3509 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3510 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3511 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3512 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3513 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3514 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3515 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3516 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3517 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3518 ;
3519 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3520 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3521 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3522 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3523 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3524 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3525 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3526 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3527 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3528 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3529 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3530 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3531 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3532 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3533 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3534 ;
3535 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3536 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3537 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3538 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3539 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3540 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3541 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3542 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3543 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3544 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3545 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3546 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3547 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3548 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3549 ; GFX940-TGSPLIT-NEXT: s_endpgm
3550 ;
3551 ; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3552 ; GFX11-WGP: ; %bb.0: ; %entry
3553 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3554 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3555 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3556 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3557 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3558 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3559 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3560 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3561 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3562 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3563 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3564 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3565 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3566 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3567 ; GFX11-WGP-NEXT: s_endpgm
3568 ;
3569 ; GFX11-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3570 ; GFX11-CU: ; %bb.0: ; %entry
3571 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3572 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3573 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3574 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3575 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3576 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3577 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3578 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3579 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3580 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3581 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3582 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3583 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3584 ; GFX11-CU-NEXT: s_endpgm
3585 ;
3586 ; GFX12-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3587 ; GFX12-WGP: ; %bb.0: ; %entry
3588 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3589 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3590 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3591 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3592 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3593 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3594 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3595 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3596 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3597 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3598 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3599 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
3600 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3601 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3602 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3603 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3604 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
3605 ; GFX12-WGP-NEXT: s_endpgm
3606 ;
3607 ; GFX12-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
3608 ; GFX12-CU: ; %bb.0: ; %entry
3609 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3610 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3611 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3612 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3613 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3614 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3615 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3616 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3617 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3618 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3619 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3620 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3621 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3622 ; GFX12-CU-NEXT: s_endpgm
3623 ptr %out, i32 %in, i32 %old) {
3624 entry:
3625 %gep = getelementptr i32, ptr %out, i32 4
3626 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
3627 ret void
3628 }
3630 define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
3631 ; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3632 ; GFX7: ; %bb.0: ; %entry
3633 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3634 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3635 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3636 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3637 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3638 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3639 ; GFX7-NEXT: s_mov_b32 s4, s8
3640 ; GFX7-NEXT: s_mov_b32 s5, s9
3641 ; GFX7-NEXT: s_mov_b32 s9, s10
3642 ; GFX7-NEXT: s_mov_b32 s8, s11
3643 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3644 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3645 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3646 ; GFX7-NEXT: s_mov_b32 s5, s8
3647 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3648 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3649 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3650 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3651 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3652 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3653 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3654 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3655 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3656 ; GFX7-NEXT: s_endpgm
3657 ;
3658 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3659 ; GFX10-WGP: ; %bb.0: ; %entry
3660 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
3661 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3662 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3663 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3664 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3665 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3666 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3667 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3668 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3669 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3670 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3671 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3672 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3673 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3674 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3675 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3676 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3677 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3678 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3679 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3680 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3681 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3682 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3683 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3684 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3685 ; GFX10-WGP-NEXT: buffer_gl0_inv
3686 ; GFX10-WGP-NEXT: s_endpgm
3687 ;
3688 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3689 ; GFX10-CU: ; %bb.0: ; %entry
3690 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
3691 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3692 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3693 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3694 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3695 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3696 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3697 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3698 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3699 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3700 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3701 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3702 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3703 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3704 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3705 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3706 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3707 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3708 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3709 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3710 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3711 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3712 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3713 ; GFX10-CU-NEXT: s_endpgm
3714 ;
3715 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3716 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3717 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
3718 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3719 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3720 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3721 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3722 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3723 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3724 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3725 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3726 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3727 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3728 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3729 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3730 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
3732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
3733 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3734 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
3735 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3737 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3738 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3739 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3740 ; SKIP-CACHE-INV-NEXT: s_endpgm
3741 ;
3742 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3743 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3744 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3745 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3746 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3747 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3748 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3749 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3750 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3751 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3752 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3753 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3754 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3755 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3756 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3757 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3758 ;
3759 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3760 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3761 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3762 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3763 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3764 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3765 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3766 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3767 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
3768 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3769 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3770 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
3771 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3772 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3773 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3774 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3775 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3776 ;
3777 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3778 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3779 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3780 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3781 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3782 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3783 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3784 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3785 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3786 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3787 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3788 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3789 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3790 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3791 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3792 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3793 ;
3794 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3795 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3796 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3797 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3798 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3799 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3800 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3801 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3802 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
3803 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3804 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
3805 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3806 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3807 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3808 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3809 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
3810 ; GFX940-TGSPLIT-NEXT: s_endpgm
3811 ;
3812 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3813 ; GFX11-WGP: ; %bb.0: ; %entry
3814 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3815 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3816 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3817 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3818 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3819 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
3820 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
3821 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3822 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
3823 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
3824 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
3825 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3826 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3827 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3828 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3829 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3830 ; GFX11-WGP-NEXT: buffer_gl0_inv
3831 ; GFX11-WGP-NEXT: s_endpgm
3832 ;
3833 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3834 ; GFX11-CU: ; %bb.0: ; %entry
3835 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3836 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3837 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3838 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3839 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3840 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
3841 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
3842 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3843 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
3844 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
3845 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
3846 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3847 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3848 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3849 ; GFX11-CU-NEXT: s_endpgm
3850 ;
3851 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3852 ; GFX12-WGP: ; %bb.0: ; %entry
3853 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3854 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3855 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3856 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3857 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3858 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
3859 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
3860 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3861 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
3862 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
3863 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
3864 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
3865 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3866 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3867 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3868 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3869 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
3870 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
3871 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
3872 ; GFX12-WGP-NEXT: s_endpgm
3873 ;
3874 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
3875 ; GFX12-CU: ; %bb.0: ; %entry
3876 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3877 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3878 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3879 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3880 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3881 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
3882 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
3883 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3884 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
3885 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
3886 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
3887 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3888 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3889 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3890 ; GFX12-CU-NEXT: s_endpgm
3891 ptr %out, i32 %in, i32 %old) {
3892 entry:
3893 %gep = getelementptr i32, ptr %out, i32 4
3894 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
3895 ret void
3896 }
3898 define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
3899 ; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
3900 ; GFX7: ; %bb.0: ; %entry
3901 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3902 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3903 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3904 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3905 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3906 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3907 ; GFX7-NEXT: s_mov_b32 s4, s8
3908 ; GFX7-NEXT: s_mov_b32 s5, s9
3909 ; GFX7-NEXT: s_mov_b32 s9, s10
3910 ; GFX7-NEXT: s_mov_b32 s8, s11
3911 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3912 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3913 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3914 ; GFX7-NEXT: s_mov_b32 s5, s8
3915 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3916 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3917 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3918 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3919 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3920 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3921 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3922 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3923 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3924 ; GFX7-NEXT: s_endpgm
3925 ;
3926 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
3927 ; GFX10-WGP: ; %bb.0: ; %entry
3928 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
3929 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3930 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
3931 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
3932 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
3933 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3934 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
3935 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
3936 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
3937 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
3938 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
3939 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
3940 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3941 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
3942 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
3943 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
3944 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3945 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
3946 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3947 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3948 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3949 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3950 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3951 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3952 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3953 ; GFX10-WGP-NEXT: buffer_gl0_inv
3954 ; GFX10-WGP-NEXT: s_endpgm
3955 ;
3956 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
3957 ; GFX10-CU: ; %bb.0: ; %entry
3958 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
3959 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3960 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
3961 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
3962 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
3963 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3964 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
3965 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
3966 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
3967 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
3968 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
3969 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
3970 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3971 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
3972 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
3973 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
3974 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3975 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
3976 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3977 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3978 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3979 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3980 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3981 ; GFX10-CU-NEXT: s_endpgm
3982 ;
3983 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
3984 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3985 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
3986 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
3987 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
3988 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
3989 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
3990 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3991 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
3992 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
3993 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
3994 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
3995 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
3996 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
3997 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
3998 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
3999 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4000 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4001 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4002 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4003 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4004 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4005 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4006 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4007 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4008 ; SKIP-CACHE-INV-NEXT: s_endpgm
4009 ;
4010 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4011 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4012 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4013 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4014 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4015 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4016 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4017 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4018 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4019 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4020 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4021 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4022 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4023 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4024 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4025 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4026 ;
4027 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4028 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4029 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4030 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4031 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4032 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4033 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4034 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4035 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4036 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4037 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4038 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4039 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4040 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4041 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4042 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4043 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4044 ;
4045 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4046 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4047 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4048 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4049 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4050 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4051 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4052 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4053 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4054 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4055 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4056 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4057 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4058 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4059 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4060 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4061 ;
4062 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4063 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4064 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4065 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4066 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4067 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4068 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4069 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4070 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4071 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4072 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4073 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4074 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4075 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4076 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4077 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4078 ; GFX940-TGSPLIT-NEXT: s_endpgm
4079 ;
4080 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4081 ; GFX11-WGP: ; %bb.0: ; %entry
4082 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4083 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4084 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4085 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4086 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4087 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4088 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4089 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4090 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4091 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4092 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4093 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4094 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4095 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4096 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4097 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4098 ; GFX11-WGP-NEXT: buffer_gl0_inv
4099 ; GFX11-WGP-NEXT: s_endpgm
4100 ;
4101 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4102 ; GFX11-CU: ; %bb.0: ; %entry
4103 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4104 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4105 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4106 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4107 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4108 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4109 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4110 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4111 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4112 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4113 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4114 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4115 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4116 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4117 ; GFX11-CU-NEXT: s_endpgm
4118 ;
4119 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4120 ; GFX12-WGP: ; %bb.0: ; %entry
4121 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4122 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4123 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4124 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4125 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4126 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4127 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4128 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4129 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4130 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4131 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4132 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
4133 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4134 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4135 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4136 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4137 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
4138 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4139 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4140 ; GFX12-WGP-NEXT: s_endpgm
4141 ;
4142 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
4143 ; GFX12-CU: ; %bb.0: ; %entry
4144 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4145 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4146 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4147 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4148 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4149 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4150 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4151 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4152 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4153 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4154 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4155 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4156 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4157 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4158 ; GFX12-CU-NEXT: s_endpgm
4159 ptr %out, i32 %in, i32 %old) {
4160 entry:
4161 %gep = getelementptr i32, ptr %out, i32 4
4162 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
4163 ret void
4164 }
4166 define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
4167 ; GFX7-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4168 ; GFX7: ; %bb.0: ; %entry
4169 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4170 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4171 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4172 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4173 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4174 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4175 ; GFX7-NEXT: s_mov_b32 s4, s8
4176 ; GFX7-NEXT: s_mov_b32 s5, s9
4177 ; GFX7-NEXT: s_mov_b32 s9, s10
4178 ; GFX7-NEXT: s_mov_b32 s8, s11
4179 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4180 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4181 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4182 ; GFX7-NEXT: s_mov_b32 s5, s8
4183 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4184 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4185 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4186 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4187 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4188 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4189 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4190 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4191 ; GFX7-NEXT: s_endpgm
4192 ;
4193 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4194 ; GFX10-WGP: ; %bb.0: ; %entry
4195 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
4196 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4197 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4198 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4199 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4200 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4201 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4202 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4203 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4204 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4205 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4206 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4207 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4208 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4209 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4210 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4211 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4212 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4213 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4214 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4215 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4216 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4217 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4218 ; GFX10-WGP-NEXT: buffer_gl0_inv
4219 ; GFX10-WGP-NEXT: s_endpgm
4220 ;
4221 ; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4222 ; GFX10-CU: ; %bb.0: ; %entry
4223 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
4224 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4225 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4226 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4227 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4228 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4229 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4230 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4231 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4232 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4233 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4234 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4235 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4236 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4237 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4238 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4239 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4240 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4241 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4242 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4243 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4244 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4245 ; GFX10-CU-NEXT: s_endpgm
4246 ;
4247 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4248 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4249 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
4250 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4251 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4252 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4253 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4254 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4255 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4256 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4257 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4258 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4259 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4260 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4261 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4262 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4263 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4264 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4265 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4266 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4267 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4268 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4269 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4270 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4271 ; SKIP-CACHE-INV-NEXT: s_endpgm
4272 ;
4273 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4274 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4275 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4276 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4277 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4278 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4279 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4280 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4281 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4282 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4283 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4284 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4285 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4286 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4287 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4288 ;
4289 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4290 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4291 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4292 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4293 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4294 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4295 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4296 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4297 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4298 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4299 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4300 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4301 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4302 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4303 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4304 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4305 ;
4306 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4307 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4308 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4309 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4310 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4311 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4312 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4313 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4314 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4315 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4316 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4317 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4318 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4319 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4320 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4321 ;
4322 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4323 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4324 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4325 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4326 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4327 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4328 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4329 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4330 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4331 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4332 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4333 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4334 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4335 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4336 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4337 ; GFX940-TGSPLIT-NEXT: s_endpgm
4338 ;
4339 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4340 ; GFX11-WGP: ; %bb.0: ; %entry
4341 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4342 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4343 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4344 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4345 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4346 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4347 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4348 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4349 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4350 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4351 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4352 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4353 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4354 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4355 ; GFX11-WGP-NEXT: buffer_gl0_inv
4356 ; GFX11-WGP-NEXT: s_endpgm
4357 ;
4358 ; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4359 ; GFX11-CU: ; %bb.0: ; %entry
4360 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4361 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4362 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4363 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4364 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4365 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4366 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4367 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4368 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4369 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4370 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4371 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4372 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4373 ; GFX11-CU-NEXT: s_endpgm
4374 ;
4375 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4376 ; GFX12-WGP: ; %bb.0: ; %entry
4377 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4378 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4379 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4380 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4381 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4382 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4383 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4384 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4385 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4386 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4387 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4388 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
4389 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4390 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4391 ; GFX12-WGP-NEXT: s_endpgm
4392 ;
4393 ; GFX12-CU-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
4394 ; GFX12-CU: ; %bb.0: ; %entry
4395 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4396 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4397 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4398 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4399 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4400 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4401 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4402 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4403 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4404 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4405 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4406 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4407 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4408 ; GFX12-CU-NEXT: s_endpgm
4409 ptr %out, i32 %in, i32 %old) {
4410 entry:
4411 %gep = getelementptr i32, ptr %out, i32 4
4412 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
4413 ret void
4414 }
4416 define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
4417 ; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4418 ; GFX7: ; %bb.0: ; %entry
4419 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4420 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4421 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4422 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4423 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4424 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4425 ; GFX7-NEXT: s_mov_b32 s4, s8
4426 ; GFX7-NEXT: s_mov_b32 s5, s9
4427 ; GFX7-NEXT: s_mov_b32 s9, s10
4428 ; GFX7-NEXT: s_mov_b32 s8, s11
4429 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4430 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4431 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4432 ; GFX7-NEXT: s_mov_b32 s5, s8
4433 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4434 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4435 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4436 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4437 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4438 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4439 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4440 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4441 ; GFX7-NEXT: s_endpgm
4442 ;
4443 ; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4444 ; GFX10-WGP: ; %bb.0: ; %entry
4445 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
4446 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4447 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4448 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4449 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4450 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4451 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4452 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4453 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4454 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4455 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4456 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4457 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4458 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4459 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4460 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4461 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4462 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4463 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4464 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4465 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4466 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4467 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4468 ; GFX10-WGP-NEXT: buffer_gl0_inv
4469 ; GFX10-WGP-NEXT: s_endpgm
4470 ;
4471 ; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4472 ; GFX10-CU: ; %bb.0: ; %entry
4473 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
4474 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4475 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4476 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4477 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4478 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4479 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4480 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4481 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4482 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4483 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4484 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4485 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4486 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4487 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4488 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4489 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4490 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4491 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4492 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4493 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4494 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4495 ; GFX10-CU-NEXT: s_endpgm
4496 ;
4497 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4498 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4499 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
4500 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4501 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4502 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4503 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4504 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4505 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4506 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4507 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4508 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4509 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4510 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4511 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4512 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4513 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4514 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4515 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4516 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4517 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4518 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4519 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4520 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4521 ; SKIP-CACHE-INV-NEXT: s_endpgm
4522 ;
4523 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4524 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4525 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4526 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4527 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4528 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4529 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4530 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4531 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4532 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4533 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4534 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4535 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4536 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4537 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4538 ;
4539 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4540 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4541 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4542 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4543 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4544 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4545 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4546 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4547 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4548 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4549 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4550 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4551 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4552 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4553 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4554 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4555 ;
4556 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4557 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4558 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4559 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4560 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4561 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4562 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4563 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4564 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4565 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4566 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4567 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4568 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4569 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4570 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4571 ;
4572 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4573 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4574 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4575 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4576 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4577 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4578 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4579 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4580 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4581 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4582 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4583 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4584 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4585 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4586 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4587 ; GFX940-TGSPLIT-NEXT: s_endpgm
4588 ;
4589 ; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4590 ; GFX11-WGP: ; %bb.0: ; %entry
4591 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4592 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4593 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4594 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4595 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4596 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4597 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4598 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4599 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4600 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4601 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4602 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4603 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4604 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4605 ; GFX11-WGP-NEXT: buffer_gl0_inv
4606 ; GFX11-WGP-NEXT: s_endpgm
4607 ;
4608 ; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4609 ; GFX11-CU: ; %bb.0: ; %entry
4610 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4611 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4612 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4613 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4614 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4615 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4616 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4617 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4618 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4619 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4620 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4621 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4622 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4623 ; GFX11-CU-NEXT: s_endpgm
4624 ;
4625 ; GFX12-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4626 ; GFX12-WGP: ; %bb.0: ; %entry
4627 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4628 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4629 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4630 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4631 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4632 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4633 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4634 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4635 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4636 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4637 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4638 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
4639 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4640 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4641 ; GFX12-WGP-NEXT: s_endpgm
4642 ;
4643 ; GFX12-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
4644 ; GFX12-CU: ; %bb.0: ; %entry
4645 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4646 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4647 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4648 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4649 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4650 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4651 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4652 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4653 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4654 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4655 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4656 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4657 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4658 ; GFX12-CU-NEXT: s_endpgm
4659 ptr %out, i32 %in, i32 %old) {
4660 entry:
4661 %gep = getelementptr i32, ptr %out, i32 4
4662 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
4663 ret void
4664 }
4666 define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
4667 ; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
4668 ; GFX7: ; %bb.0: ; %entry
4669 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4670 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4671 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4672 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4673 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4674 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4675 ; GFX7-NEXT: s_mov_b32 s4, s8
4676 ; GFX7-NEXT: s_mov_b32 s5, s9
4677 ; GFX7-NEXT: s_mov_b32 s9, s10
4678 ; GFX7-NEXT: s_mov_b32 s8, s11
4679 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4680 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4681 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4682 ; GFX7-NEXT: s_mov_b32 s5, s8
4683 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4684 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4685 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4686 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4687 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4688 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4689 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4690 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4691 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4692 ; GFX7-NEXT: s_endpgm
4693 ;
4694 ; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
4695 ; GFX10-WGP: ; %bb.0: ; %entry
4696 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
4697 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4698 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4699 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4700 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4701 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4702 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4703 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4704 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4705 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4706 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4707 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4708 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4709 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4710 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4711 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4712 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4713 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4714 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4715 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4716 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4717 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4718 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4719 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4720 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4721 ; GFX10-WGP-NEXT: buffer_gl0_inv
4722 ; GFX10-WGP-NEXT: s_endpgm
4723 ;
4724 ; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
4725 ; GFX10-CU: ; %bb.0: ; %entry
4726 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
4727 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4728 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4729 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4730 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4731 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4732 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
4733 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
4734 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
4735 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
4736 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
4737 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
4738 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4739 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
4740 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
4741 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
4742 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4743 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
4744 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4745 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4746 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4747 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4748 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4749 ; GFX10-CU-NEXT: s_endpgm
4750 ;
4751 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
4752 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4753 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
4754 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
4755 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
4756 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
4757 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
4758 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4759 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
4760 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
4761 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
4762 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
4763 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
4764 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
4765 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
4766 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
4767 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
4768 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
4769 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4770 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
4771 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4772 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4773 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4774 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4775 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4776 ; SKIP-CACHE-INV-NEXT: s_endpgm
4777 ;
4778 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
4779 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4780 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4781 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4782 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4783 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4784 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4785 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4786 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4787 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4788 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4789 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4790 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4791 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4792 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4793 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4794 ;
4795 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
4796 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4797 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4798 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4799 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4800 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4801 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4802 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4803 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
4804 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4805 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4806 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
4807 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4808 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4809 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4810 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4811 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4812 ;
4813 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
4814 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4815 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4816 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4817 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4818 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4819 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4820 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4821 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4822 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4823 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4824 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4825 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4826 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4827 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4828 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4829 ;
4830 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_cmpxchg:
4831 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4832 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4833 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4834 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4835 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4836 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4837 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4838 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
4839 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4840 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
4841 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4842 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4843 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
4844 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4845 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4846 ; GFX940-TGSPLIT-NEXT: s_endpgm
4847 ;
4848 ; GFX11-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
4849 ; GFX11-WGP: ; %bb.0: ; %entry
4850 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4851 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4852 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4853 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4854 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4855 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
4856 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
4857 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4858 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
4859 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
4860 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
4861 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4862 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4863 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4864 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4865 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4866 ; GFX11-WGP-NEXT: buffer_gl0_inv
4867 ; GFX11-WGP-NEXT: s_endpgm
4868 ;
4869 ; GFX11-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
4870 ; GFX11-CU: ; %bb.0: ; %entry
4871 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4872 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4873 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4874 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4875 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4876 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
4877 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
4878 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4879 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
4880 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
4881 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
4882 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4883 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4884 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4885 ; GFX11-CU-NEXT: s_endpgm
4886 ;
4887 ; GFX12-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
4888 ; GFX12-WGP: ; %bb.0: ; %entry
4889 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4890 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4891 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4892 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4893 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4894 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
4895 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
4896 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4897 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
4898 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
4899 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
4900 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
4901 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4902 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4903 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4904 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4905 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
4906 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
4907 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4908 ; GFX12-WGP-NEXT: s_endpgm
4909 ;
4910 ; GFX12-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
4911 ; GFX12-CU: ; %bb.0: ; %entry
4912 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4913 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4914 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4915 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4916 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4917 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
4918 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
4919 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4920 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
4921 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
4922 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
4923 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4924 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
4925 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4926 ; GFX12-CU-NEXT: s_endpgm
4927 ptr %out, i32 %in, i32 %old) {
4928 entry:
4929 %gep = getelementptr i32, ptr %out, i32 4
4930 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
4931 ret void
4932 }
4934 define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
4935 ; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
4936 ; GFX7: ; %bb.0: ; %entry
4937 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4938 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4939 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4940 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4941 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4942 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4943 ; GFX7-NEXT: s_mov_b32 s4, s8
4944 ; GFX7-NEXT: s_mov_b32 s5, s9
4945 ; GFX7-NEXT: s_mov_b32 s9, s10
4946 ; GFX7-NEXT: s_mov_b32 s8, s11
4947 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4948 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4949 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4950 ; GFX7-NEXT: s_mov_b32 s5, s8
4951 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4952 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4953 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4954 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4955 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4956 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4957 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4958 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4959 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4960 ; GFX7-NEXT: s_endpgm
4961 ;
4962 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
4963 ; GFX10-WGP: ; %bb.0: ; %entry
4964 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
4965 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4966 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
4967 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
4968 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
4969 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4970 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
4971 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
4972 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
4973 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
4974 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
4975 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
4976 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4977 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
4978 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
4979 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
4980 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4981 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
4982 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4983 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4984 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4985 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4986 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4987 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4988 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4989 ; GFX10-WGP-NEXT: buffer_gl0_inv
4990 ; GFX10-WGP-NEXT: s_endpgm
4991 ;
4992 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
4993 ; GFX10-CU: ; %bb.0: ; %entry
4994 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
4995 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4996 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
4997 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
4998 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
4999 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5000 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5001 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5002 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5003 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5004 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5005 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5006 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5007 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5008 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5009 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5010 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5011 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5012 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5013 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5014 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5015 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5016 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5017 ; GFX10-CU-NEXT: s_endpgm
5018 ;
5019 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5020 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5021 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
5022 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5023 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5024 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5025 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5026 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5027 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5028 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5029 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5030 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5031 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5032 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5033 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5034 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5035 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5036 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5037 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5038 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5039 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5040 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5041 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5042 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5043 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5044 ; SKIP-CACHE-INV-NEXT: s_endpgm
5045 ;
5046 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5047 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5048 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5049 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5050 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5051 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5052 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5053 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5054 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5055 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5056 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5057 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5058 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5059 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5060 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5061 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5062 ;
5063 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5064 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5065 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5066 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5067 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5068 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5069 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5070 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5071 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5072 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5073 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5074 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5075 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5076 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5077 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5078 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5079 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5080 ;
5081 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5082 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5083 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5084 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5085 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5086 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5087 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5088 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5089 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5090 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5091 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5092 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5093 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5094 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5095 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5096 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5097 ;
5098 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5099 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5100 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5101 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5102 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5103 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5104 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5105 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5106 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5107 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5108 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5109 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5110 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5111 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5112 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5113 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5114 ; GFX940-TGSPLIT-NEXT: s_endpgm
5115 ;
5116 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5117 ; GFX11-WGP: ; %bb.0: ; %entry
5118 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5119 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5120 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5121 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5122 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5123 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5124 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5125 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5127 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5128 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5129 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5130 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5131 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5132 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5133 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5134 ; GFX11-WGP-NEXT: buffer_gl0_inv
5135 ; GFX11-WGP-NEXT: s_endpgm
5136 ;
5137 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5138 ; GFX11-CU: ; %bb.0: ; %entry
5139 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5140 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5141 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5142 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5143 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5144 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5145 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5146 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5147 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5148 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5149 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5150 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5151 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5152 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5153 ; GFX11-CU-NEXT: s_endpgm
5154 ;
5155 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5156 ; GFX12-WGP: ; %bb.0: ; %entry
5157 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5158 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5159 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5160 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5161 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5162 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5163 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5164 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5166 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5167 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5168 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5169 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5170 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5171 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5172 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5173 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
5174 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5175 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5176 ; GFX12-WGP-NEXT: s_endpgm
5177 ;
5178 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
5179 ; GFX12-CU: ; %bb.0: ; %entry
5180 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5181 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5182 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5183 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5184 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5185 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5186 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5187 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5188 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5189 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5190 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5191 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5192 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5193 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5194 ; GFX12-CU-NEXT: s_endpgm
5195 ptr %out, i32 %in, i32 %old) {
5196 entry:
5197 %gep = getelementptr i32, ptr %out, i32 4
5198 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
5199 ret void
5200 }
5202 define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
5203 ; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5204 ; GFX7: ; %bb.0: ; %entry
5205 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5206 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5207 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5208 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5209 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5210 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5211 ; GFX7-NEXT: s_mov_b32 s4, s8
5212 ; GFX7-NEXT: s_mov_b32 s5, s9
5213 ; GFX7-NEXT: s_mov_b32 s9, s10
5214 ; GFX7-NEXT: s_mov_b32 s8, s11
5215 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5216 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5217 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5218 ; GFX7-NEXT: s_mov_b32 s5, s8
5219 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5220 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5221 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5222 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5223 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5224 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5225 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5226 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5227 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5228 ; GFX7-NEXT: s_endpgm
5229 ;
5230 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5231 ; GFX10-WGP: ; %bb.0: ; %entry
5232 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
5233 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5234 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
5235 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
5236 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
5237 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5238 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
5239 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
5240 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
5241 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
5242 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
5243 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
5244 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5245 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
5246 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
5247 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5248 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5249 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5250 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5251 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5252 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5253 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5254 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5255 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5256 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5257 ; GFX10-WGP-NEXT: buffer_gl0_inv
5258 ; GFX10-WGP-NEXT: s_endpgm
5259 ;
5260 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5261 ; GFX10-CU: ; %bb.0: ; %entry
5262 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
5263 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5264 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
5265 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
5266 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
5267 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5268 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5269 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5270 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5271 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5272 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5273 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5274 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5275 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5276 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5277 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5278 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5279 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5280 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5281 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5282 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5283 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5284 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5285 ; GFX10-CU-NEXT: s_endpgm
5286 ;
5287 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5288 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5289 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
5290 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5291 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5292 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5293 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5294 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5296 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5297 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5298 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5299 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5300 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5301 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5302 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5304 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5305 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5309 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5310 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5311 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5312 ; SKIP-CACHE-INV-NEXT: s_endpgm
5313 ;
5314 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5315 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5316 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5317 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5318 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5319 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5320 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5321 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5322 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5323 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5324 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5325 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5326 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5327 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5328 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5329 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5330 ;
5331 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5332 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5333 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5334 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5335 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5336 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5337 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5338 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5339 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5340 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5341 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5342 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5343 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5344 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5345 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5346 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5347 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5348 ;
5349 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5350 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5351 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5352 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5353 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5354 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5355 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5357 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5358 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5359 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5360 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5361 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5362 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5363 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5364 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5365 ;
5366 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5367 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5368 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5369 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5370 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5371 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5372 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5373 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5374 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5375 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5376 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5377 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5378 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5379 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5380 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5381 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5382 ; GFX940-TGSPLIT-NEXT: s_endpgm
5383 ;
5384 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5385 ; GFX11-WGP: ; %bb.0: ; %entry
5386 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5387 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5388 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5389 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5390 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5391 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5392 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5393 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5394 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5395 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5396 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5397 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5398 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5399 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5400 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5401 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5402 ; GFX11-WGP-NEXT: buffer_gl0_inv
5403 ; GFX11-WGP-NEXT: s_endpgm
5404 ;
5405 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5406 ; GFX11-CU: ; %bb.0: ; %entry
5407 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5408 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5409 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5410 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5411 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5412 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5413 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5414 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5415 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5416 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5417 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5418 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5419 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5420 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5421 ; GFX11-CU-NEXT: s_endpgm
5422 ;
5423 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5424 ; GFX12-WGP: ; %bb.0: ; %entry
5425 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5426 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5427 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5428 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5429 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5430 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5431 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5432 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5433 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5434 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5436 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5437 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5438 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5439 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5440 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5441 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
5442 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5443 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5444 ; GFX12-WGP-NEXT: s_endpgm
5445 ;
5446 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
5447 ; GFX12-CU: ; %bb.0: ; %entry
5448 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5449 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5450 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5451 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5452 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5453 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5454 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5455 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5456 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5457 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5458 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5459 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5460 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5461 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5462 ; GFX12-CU-NEXT: s_endpgm
5463 ptr %out, i32 %in, i32 %old) {
5464 entry:
5465 %gep = getelementptr i32, ptr %out, i32 4
5466 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
5467 ret void
5468 }
5470 define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
5471 ; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5472 ; GFX7: ; %bb.0: ; %entry
5473 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5474 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5475 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5476 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5477 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5478 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5479 ; GFX7-NEXT: s_mov_b32 s4, s8
5480 ; GFX7-NEXT: s_mov_b32 s5, s9
5481 ; GFX7-NEXT: s_mov_b32 s9, s10
5482 ; GFX7-NEXT: s_mov_b32 s8, s11
5483 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5484 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5485 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5486 ; GFX7-NEXT: s_mov_b32 s5, s8
5487 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5488 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5489 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5490 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5491 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5492 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5493 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5494 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5495 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5496 ; GFX7-NEXT: s_endpgm
5497 ;
5498 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5499 ; GFX10-WGP: ; %bb.0: ; %entry
5500 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
5501 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5502 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
5503 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
5504 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
5505 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5506 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
5507 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
5508 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
5509 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
5510 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
5511 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
5512 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5513 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
5514 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
5515 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5516 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5517 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5518 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5519 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5520 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5521 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5522 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5523 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5524 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5525 ; GFX10-WGP-NEXT: buffer_gl0_inv
5526 ; GFX10-WGP-NEXT: s_endpgm
5527 ;
5528 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5529 ; GFX10-CU: ; %bb.0: ; %entry
5530 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
5531 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5532 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
5533 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
5534 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
5535 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5536 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
5537 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
5538 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
5539 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
5540 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
5541 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
5542 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5543 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
5544 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
5545 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5546 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5547 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5548 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5549 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5550 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5551 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5552 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5553 ; GFX10-CU-NEXT: s_endpgm
5554 ;
5555 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5556 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5557 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
5558 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
5559 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
5560 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
5561 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
5562 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5563 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
5564 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
5565 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
5566 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
5567 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
5568 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
5569 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
5570 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
5571 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
5572 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5573 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5574 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5575 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5576 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5577 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5578 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5579 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5580 ; SKIP-CACHE-INV-NEXT: s_endpgm
5581 ;
5582 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5583 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5584 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5585 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5586 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5587 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5588 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5589 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5590 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5591 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5592 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5593 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5594 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5595 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5596 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5597 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5598 ;
5599 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5600 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5601 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5602 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5603 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5604 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5605 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5606 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5607 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5608 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5609 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5610 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5611 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5612 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5613 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5614 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5615 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5616 ;
5617 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5618 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5619 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5620 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5621 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5622 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5623 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5624 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5625 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5626 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5627 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5628 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5629 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5630 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5631 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5632 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5633 ;
5634 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5635 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5636 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5637 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5638 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5639 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5640 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5641 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5642 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5643 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5644 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5645 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5646 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5647 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
5648 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5649 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5650 ; GFX940-TGSPLIT-NEXT: s_endpgm
5651 ;
5652 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5653 ; GFX11-WGP: ; %bb.0: ; %entry
5654 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5655 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5656 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5657 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5658 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5659 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5660 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5661 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5662 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5663 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5664 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5665 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5666 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5667 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5668 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5669 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5670 ; GFX11-WGP-NEXT: buffer_gl0_inv
5671 ; GFX11-WGP-NEXT: s_endpgm
5672 ;
5673 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5674 ; GFX11-CU: ; %bb.0: ; %entry
5675 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5676 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5677 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5678 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5679 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5680 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5681 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5682 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5683 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5684 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5685 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5686 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5687 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5688 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5689 ; GFX11-CU-NEXT: s_endpgm
5690 ;
5691 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5692 ; GFX12-WGP: ; %bb.0: ; %entry
5693 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5694 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5695 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5696 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5697 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5698 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5699 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5700 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5701 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5702 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5703 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5704 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5705 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5706 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5707 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5708 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5709 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
5710 ; GFX12-WGP-NEXT: s_wait_storecnt_dscnt 0x0
5711 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5712 ; GFX12-WGP-NEXT: s_endpgm
5713 ;
5714 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
5715 ; GFX12-CU: ; %bb.0: ; %entry
5716 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5717 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5718 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5719 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5720 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5721 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5722 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5723 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5724 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5725 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5726 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5727 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5728 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
5729 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5730 ; GFX12-CU-NEXT: s_endpgm
5731 ptr %out, i32 %in, i32 %old) {
5732 entry:
5733 %gep = getelementptr i32, ptr %out, i32 4
5734 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
5735 ret void
5736 }
5738 define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
5739 ; GFX7-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5740 ; GFX7: ; %bb.0: ; %entry
5741 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5742 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
5743 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
5744 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
5745 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5746 ; GFX7-NEXT: s_mov_b32 s6, s4
5747 ; GFX7-NEXT: s_mov_b32 s7, s5
5748 ; GFX7-NEXT: s_mov_b32 s11, s12
5749 ; GFX7-NEXT: s_mov_b32 s10, s13
5750 ; GFX7-NEXT: s_add_u32 s6, s6, s11
5751 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
5752 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
5753 ; GFX7-NEXT: s_mov_b32 s7, s10
5754 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
5755 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
5756 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5757 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5758 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5759 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
5760 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5761 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5762 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5763 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5764 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5765 ; GFX7-NEXT: s_endpgm
5766 ;
5767 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5768 ; GFX10-WGP: ; %bb.0: ; %entry
5769 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5770 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
5771 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
5772 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
5773 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5774 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
5775 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
5776 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
5777 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
5778 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
5779 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
5780 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
5781 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
5782 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
5783 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
5784 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5785 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
5786 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
5787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
5788 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5789 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5791 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5792 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5793 ; GFX10-WGP-NEXT: s_endpgm
5794 ;
5795 ; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5796 ; GFX10-CU: ; %bb.0: ; %entry
5797 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5798 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
5799 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
5800 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
5801 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5802 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
5803 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
5804 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
5805 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
5806 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
5807 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
5808 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
5809 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
5810 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
5811 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
5812 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5813 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
5814 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
5815 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
5816 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5817 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5818 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5819 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5820 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5821 ; GFX10-CU-NEXT: s_endpgm
5822 ;
5823 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5824 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5825 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
5826 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
5827 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
5828 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
5829 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5830 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
5831 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
5832 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
5833 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
5834 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
5835 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
5836 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
5837 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
5838 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
5839 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5840 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5841 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
5842 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5843 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
5844 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5845 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5846 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5847 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5848 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5849 ; SKIP-CACHE-INV-NEXT: s_endpgm
5850 ;
5851 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5852 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5853 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5854 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5855 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5856 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5857 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5858 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5859 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5860 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5861 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5862 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5863 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5864 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5865 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5866 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5867 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5868 ;
5869 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5870 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5871 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5872 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5873 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5874 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5875 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5876 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5877 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
5878 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5879 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5880 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5881 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5882 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
5883 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5884 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5885 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5886 ;
5887 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5888 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5889 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5890 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5891 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5892 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5893 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5894 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5895 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5896 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5897 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5898 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5899 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5900 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5901 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5902 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5903 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5904 ;
5905 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5906 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5907 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5908 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5909 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5910 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5911 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5912 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5913 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5914 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5915 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
5916 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5917 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5918 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5919 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5920 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5921 ; GFX940-TGSPLIT-NEXT: s_endpgm
5922 ;
5923 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5924 ; GFX11-WGP: ; %bb.0: ; %entry
5925 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5926 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5927 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5928 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5929 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5930 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
5931 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
5932 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5933 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
5934 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5935 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5936 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5937 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
5938 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
5939 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5940 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5941 ; GFX11-WGP-NEXT: s_endpgm
5942 ;
5943 ; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5944 ; GFX11-CU: ; %bb.0: ; %entry
5945 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5946 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5947 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5948 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5949 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5950 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
5951 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
5952 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5953 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
5954 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5955 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5956 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5957 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
5958 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
5959 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5960 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5961 ; GFX11-CU-NEXT: s_endpgm
5962 ;
5963 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5964 ; GFX12-WGP: ; %bb.0: ; %entry
5965 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5966 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5967 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5968 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5969 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5970 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
5971 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
5972 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5973 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
5974 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5975 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5976 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
5977 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
5978 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
5979 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5980 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
5981 ; GFX12-WGP-NEXT: s_endpgm
5982 ;
5983 ; GFX12-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
5984 ; GFX12-CU: ; %bb.0: ; %entry
5985 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5986 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5987 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5988 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5989 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5990 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
5991 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
5992 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5993 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
5994 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5995 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5996 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
5997 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
5998 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
5999 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
6000 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
6001 ; GFX12-CU-NEXT: s_endpgm
6002 ptr %out, i32 %in, i32 %old) {
6003 entry:
6004 %gep = getelementptr i32, ptr %out, i32 4
6005 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
6006 %val0 = extractvalue { i32, i1 } %val, 0
6007 store i32 %val0, ptr %out, align 4
6008 ret void
6009 }
6011 define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
6012 ; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6013 ; GFX7: ; %bb.0: ; %entry
6014 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6015 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6016 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6017 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6018 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6019 ; GFX7-NEXT: s_mov_b32 s6, s4
6020 ; GFX7-NEXT: s_mov_b32 s7, s5
6021 ; GFX7-NEXT: s_mov_b32 s11, s12
6022 ; GFX7-NEXT: s_mov_b32 s10, s13
6023 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6024 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6025 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6026 ; GFX7-NEXT: s_mov_b32 s7, s10
6027 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6028 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6029 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6030 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6031 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6032 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6033 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6034 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6035 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6036 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6037 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6038 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6039 ; GFX7-NEXT: s_endpgm
6040 ;
6041 ; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6042 ; GFX10-WGP: ; %bb.0: ; %entry
6043 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6044 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
6045 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
6046 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
6047 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6048 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
6049 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
6050 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
6051 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
6052 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
6053 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
6054 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6055 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
6056 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
6057 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
6058 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6059 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6060 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6061 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6062 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6063 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6064 ; GFX10-WGP-NEXT: buffer_gl0_inv
6065 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6066 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6067 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6068 ; GFX10-WGP-NEXT: s_endpgm
6069 ;
6070 ; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6071 ; GFX10-CU: ; %bb.0: ; %entry
6072 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6073 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
6074 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
6075 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
6076 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6077 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
6078 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
6079 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
6080 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
6081 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
6082 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
6083 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6084 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
6085 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
6086 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
6087 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6088 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6089 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6090 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6091 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6092 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6093 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6094 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6095 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
6096 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6097 ; GFX10-CU-NEXT: s_endpgm
6098 ;
6099 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6100 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6101 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6102 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6103 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6104 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
6105 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6106 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
6107 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
6108 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
6109 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
6110 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
6111 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
6112 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
6113 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6114 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
6115 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
6116 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6117 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6118 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6119 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6120 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6121 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6122 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6123 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6124 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
6125 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6126 ; SKIP-CACHE-INV-NEXT: s_endpgm
6127 ;
6128 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6129 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6130 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6131 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6132 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6133 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6134 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6135 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6136 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6137 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6138 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6139 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6140 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6141 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6142 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6143 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6144 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6145 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6146 ;
6147 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6148 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6149 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6150 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6151 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6152 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6153 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6154 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6155 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6156 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6157 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6158 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6159 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6160 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6161 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6162 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6163 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6164 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6165 ;
6166 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6167 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6168 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6169 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6170 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6171 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6172 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6173 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6174 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6175 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6176 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6177 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6178 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6179 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6180 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6181 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6182 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6183 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6184 ;
6185 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6186 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6187 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6188 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6189 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6190 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6191 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6192 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6193 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6194 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6195 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6196 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6197 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6198 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6199 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
6200 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6201 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6202 ; GFX940-TGSPLIT-NEXT: s_endpgm
6203 ;
6204 ; GFX11-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6205 ; GFX11-WGP: ; %bb.0: ; %entry
6206 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6207 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6208 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6209 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6210 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6211 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6212 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6213 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6214 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6215 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6216 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6217 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6218 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6219 ; GFX11-WGP-NEXT: buffer_gl0_inv
6220 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6221 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6222 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6223 ; GFX11-WGP-NEXT: s_endpgm
6224 ;
6225 ; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6226 ; GFX11-CU: ; %bb.0: ; %entry
6227 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6228 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6229 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6230 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6231 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6232 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6233 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6234 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6235 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6236 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6237 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6238 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6239 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6240 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6241 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6242 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
6243 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6244 ; GFX11-CU-NEXT: s_endpgm
6245 ;
6246 ; GFX12-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6247 ; GFX12-WGP: ; %bb.0: ; %entry
6248 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6249 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6250 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6251 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6252 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6253 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6254 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6255 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6256 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6257 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6258 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6259 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
6260 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6261 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6262 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6263 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
6264 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6265 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6266 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
6267 ; GFX12-WGP-NEXT: s_endpgm
6268 ;
6269 ; GFX12-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
6270 ; GFX12-CU: ; %bb.0: ; %entry
6271 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6272 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6273 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6274 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6275 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6276 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6277 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6278 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6279 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6280 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6281 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6282 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
6283 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6284 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6285 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6286 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
6287 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
6288 ; GFX12-CU-NEXT: s_endpgm
6289 ptr %out, i32 %in, i32 %old) {
6290 entry:
6291 %gep = getelementptr i32, ptr %out, i32 4
6292 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
6293 %val0 = extractvalue { i32, i1 } %val, 0
6294 store i32 %val0, ptr %out, align 4
6295 ret void
6296 }
6298 define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
6299 ; GFX7-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6300 ; GFX7: ; %bb.0: ; %entry
6301 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6302 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6303 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6304 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6305 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6306 ; GFX7-NEXT: s_mov_b32 s6, s4
6307 ; GFX7-NEXT: s_mov_b32 s7, s5
6308 ; GFX7-NEXT: s_mov_b32 s11, s12
6309 ; GFX7-NEXT: s_mov_b32 s10, s13
6310 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6311 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6312 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6313 ; GFX7-NEXT: s_mov_b32 s7, s10
6314 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6315 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6316 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6317 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6318 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6319 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6320 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6321 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6322 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6323 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6324 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6325 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6326 ; GFX7-NEXT: s_endpgm
6327 ;
6328 ; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6329 ; GFX10-WGP: ; %bb.0: ; %entry
6330 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6331 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
6332 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
6333 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
6334 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6335 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
6336 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
6337 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
6338 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
6339 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
6340 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
6341 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6342 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
6343 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
6344 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
6345 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6346 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6347 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6348 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6349 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6350 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6351 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6352 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6353 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6354 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6355 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6356 ; GFX10-WGP-NEXT: s_endpgm
6357 ;
6358 ; GFX10-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6359 ; GFX10-CU: ; %bb.0: ; %entry
6360 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6361 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
6362 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
6363 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
6364 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6365 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
6366 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
6367 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
6368 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
6369 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
6370 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
6371 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6372 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
6373 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
6374 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
6375 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6376 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6377 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6378 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6379 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6380 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6381 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6382 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6383 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6384 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6385 ; GFX10-CU-NEXT: s_endpgm
6386 ;
6387 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6388 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6389 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6390 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6391 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6392 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
6393 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6394 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
6395 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
6396 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
6397 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
6398 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
6399 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
6400 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
6401 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6402 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
6403 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
6404 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6405 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6406 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6407 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6408 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6409 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6410 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6411 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6412 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6413 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6414 ; SKIP-CACHE-INV-NEXT: s_endpgm
6415 ;
6416 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6417 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6418 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6419 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6420 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6421 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6422 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6423 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6424 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6425 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6426 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6427 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6428 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6429 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6430 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6431 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6432 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6433 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6434 ;
6435 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6436 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6437 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6438 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6439 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6440 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6441 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6442 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6443 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6444 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6445 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6446 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6447 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6448 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6449 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6450 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6451 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6452 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6453 ;
6454 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6455 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6456 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6457 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6458 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6459 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6460 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6461 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6462 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6463 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6464 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6465 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6466 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6467 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6468 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6469 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6470 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6471 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6472 ;
6473 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6474 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6475 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6476 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6477 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6478 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6479 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6480 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6481 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6482 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6483 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6484 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6485 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6486 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6487 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6488 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6489 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6490 ; GFX940-TGSPLIT-NEXT: s_endpgm
6491 ;
6492 ; GFX11-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6493 ; GFX11-WGP: ; %bb.0: ; %entry
6494 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6495 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6496 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6497 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6498 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6499 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6500 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6501 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6502 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6503 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6504 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6505 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6506 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6507 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6508 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6509 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6510 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6511 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6512 ; GFX11-WGP-NEXT: s_endpgm
6513 ;
6514 ; GFX11-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6515 ; GFX11-CU: ; %bb.0: ; %entry
6516 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6517 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6518 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6519 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6520 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6521 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6522 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6523 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6524 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6525 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6526 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6527 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6528 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6529 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6530 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6531 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6532 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6533 ; GFX11-CU-NEXT: s_endpgm
6534 ;
6535 ; GFX12-WGP-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6536 ; GFX12-WGP: ; %bb.0: ; %entry
6537 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6538 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6539 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6540 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6541 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6542 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6543 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6544 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6545 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6546 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6547 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6548 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
6549 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6550 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6551 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6552 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6553 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
6554 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6555 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6556 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6557 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
6558 ; GFX12-WGP-NEXT: s_endpgm
6559 ;
6560 ; GFX12-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
6561 ; GFX12-CU: ; %bb.0: ; %entry
6562 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6563 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6564 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6565 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6566 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6567 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6568 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6569 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6570 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6571 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6572 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6573 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6574 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
6575 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6576 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6577 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
6578 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
6579 ; GFX12-CU-NEXT: s_endpgm
6580 ptr %out, i32 %in, i32 %old) {
6581 entry:
6582 %gep = getelementptr i32, ptr %out, i32 4
6583 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
6584 %val0 = extractvalue { i32, i1 } %val, 0
6585 store i32 %val0, ptr %out, align 4
6586 ret void
6587 }
6589 define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
6590 ; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6591 ; GFX7: ; %bb.0: ; %entry
6592 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6593 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6594 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6595 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6596 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6597 ; GFX7-NEXT: s_mov_b32 s6, s4
6598 ; GFX7-NEXT: s_mov_b32 s7, s5
6599 ; GFX7-NEXT: s_mov_b32 s11, s12
6600 ; GFX7-NEXT: s_mov_b32 s10, s13
6601 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6602 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6603 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6604 ; GFX7-NEXT: s_mov_b32 s7, s10
6605 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6606 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6607 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6608 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6609 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6610 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6611 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6612 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6613 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6614 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6615 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6616 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6617 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6618 ; GFX7-NEXT: s_endpgm
6619 ;
6620 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6621 ; GFX10-WGP: ; %bb.0: ; %entry
6622 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6623 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
6624 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
6625 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
6626 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6627 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
6628 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
6629 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
6630 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
6631 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
6632 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
6633 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6634 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
6635 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
6636 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
6637 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6638 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6639 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6640 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6641 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6642 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6643 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6644 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6645 ; GFX10-WGP-NEXT: buffer_gl0_inv
6646 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6647 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6648 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6649 ; GFX10-WGP-NEXT: s_endpgm
6650 ;
6651 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6652 ; GFX10-CU: ; %bb.0: ; %entry
6653 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6654 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
6655 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
6656 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
6657 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6658 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
6659 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
6660 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
6661 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
6662 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
6663 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
6664 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6665 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
6666 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
6667 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
6668 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6669 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6670 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6671 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6672 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6673 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6674 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6675 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6676 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6677 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
6678 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6679 ; GFX10-CU-NEXT: s_endpgm
6680 ;
6681 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6682 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6683 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6684 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6685 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6686 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
6687 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6688 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
6689 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
6690 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
6691 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
6692 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
6693 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
6694 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
6695 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6696 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
6697 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
6698 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6699 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
6700 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6701 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6702 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6703 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6704 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6705 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6706 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6707 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
6708 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6709 ; SKIP-CACHE-INV-NEXT: s_endpgm
6710 ;
6711 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6712 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6713 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6714 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6715 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6716 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6717 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6718 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6719 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6720 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6721 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6722 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6723 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6724 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6725 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6726 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6727 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6728 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6729 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6730 ;
6731 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6732 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6733 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6734 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6735 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6736 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6737 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6738 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6739 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
6740 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6741 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6742 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6743 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6744 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
6745 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6746 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6747 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
6748 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6749 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6750 ;
6751 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6752 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6753 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6754 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6755 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6756 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6757 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6758 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6759 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6760 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6761 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6762 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6763 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6764 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6765 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6766 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6767 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6768 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6769 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6770 ;
6771 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6772 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6773 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6774 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6775 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6776 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6777 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6778 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6779 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6780 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6781 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
6782 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6783 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6784 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
6785 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6786 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
6787 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
6788 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6789 ; GFX940-TGSPLIT-NEXT: s_endpgm
6790 ;
6791 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6792 ; GFX11-WGP: ; %bb.0: ; %entry
6793 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6794 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6795 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6796 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6797 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6798 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
6799 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
6800 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6801 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
6802 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6803 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6804 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6805 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6806 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6807 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6808 ; GFX11-WGP-NEXT: buffer_gl0_inv
6809 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
6810 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
6811 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6812 ; GFX11-WGP-NEXT: s_endpgm
6813 ;
6814 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6815 ; GFX11-CU: ; %bb.0: ; %entry
6816 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6817 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6818 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6819 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6820 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6821 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
6822 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
6823 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6824 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
6825 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6826 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6827 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6828 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
6829 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6830 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
6831 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
6832 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
6833 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6834 ; GFX11-CU-NEXT: s_endpgm
6835 ;
6836 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6837 ; GFX12-WGP: ; %bb.0: ; %entry
6838 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6839 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6840 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6841 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6842 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6843 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
6844 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
6845 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6846 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
6847 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6848 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6849 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
6850 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6851 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6852 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6853 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6854 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
6855 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6856 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6857 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6858 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
6859 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
6860 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
6861 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
6862 ; GFX12-WGP-NEXT: s_endpgm
6863 ;
6864 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
6865 ; GFX12-CU: ; %bb.0: ; %entry
6866 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6867 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6868 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6869 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6870 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6871 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
6872 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
6873 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6874 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
6875 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6876 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6877 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6878 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
6879 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6880 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
6881 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
6882 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
6883 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
6884 ; GFX12-CU-NEXT: s_endpgm
6885 ptr %out, i32 %in, i32 %old) {
6886 entry:
6887 %gep = getelementptr i32, ptr %out, i32 4
6888 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
6889 %val0 = extractvalue { i32, i1 } %val, 0
6890 store i32 %val0, ptr %out, align 4
6891 ret void
6892 }
6894 define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
6895 ; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
6896 ; GFX7: ; %bb.0: ; %entry
6897 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6898 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6899 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6900 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6901 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6902 ; GFX7-NEXT: s_mov_b32 s6, s4
6903 ; GFX7-NEXT: s_mov_b32 s7, s5
6904 ; GFX7-NEXT: s_mov_b32 s11, s12
6905 ; GFX7-NEXT: s_mov_b32 s10, s13
6906 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6907 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6908 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6909 ; GFX7-NEXT: s_mov_b32 s7, s10
6910 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6911 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6912 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6913 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6914 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6915 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6916 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6917 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6918 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6919 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6920 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6921 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6922 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6923 ; GFX7-NEXT: s_endpgm
6924 ;
6925 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
6926 ; GFX10-WGP: ; %bb.0: ; %entry
6927 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6928 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
6929 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
6930 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
6931 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6932 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
6933 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
6934 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
6935 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
6936 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
6937 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
6938 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6939 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
6940 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
6941 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
6942 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6943 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
6944 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
6945 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6946 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6947 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6948 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6949 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6950 ; GFX10-WGP-NEXT: buffer_gl0_inv
6951 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
6952 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
6953 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6954 ; GFX10-WGP-NEXT: s_endpgm
6955 ;
6956 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
6957 ; GFX10-CU: ; %bb.0: ; %entry
6958 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6959 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
6960 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
6961 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
6962 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6963 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
6964 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
6965 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
6966 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
6967 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
6968 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
6969 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6970 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
6971 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
6972 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
6973 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6974 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
6975 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
6976 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6977 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6978 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6979 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6980 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
6981 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
6982 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
6983 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6984 ; GFX10-CU-NEXT: s_endpgm
6985 ;
6986 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
6987 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6988 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6989 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6990 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6991 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
6992 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6993 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
6994 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
6995 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
6996 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
6997 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
6998 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
6999 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7000 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7001 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7002 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7003 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7004 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7005 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7006 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7007 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7008 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7009 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7010 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7011 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7012 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7013 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7014 ; SKIP-CACHE-INV-NEXT: s_endpgm
7015 ;
7016 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7017 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7018 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7019 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7020 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7021 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7022 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7023 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7024 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7025 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7026 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7027 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7028 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7029 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7030 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7031 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7032 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7033 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7034 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7035 ;
7036 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7037 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7038 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7039 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7040 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7041 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7042 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7043 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7044 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7045 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7046 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7047 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7048 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7049 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7050 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7051 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7052 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7053 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7054 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7055 ;
7056 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7057 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7058 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7059 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7060 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7061 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7062 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7063 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7064 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7065 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7066 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7067 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7068 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7069 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7070 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7071 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7072 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7073 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7074 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7075 ;
7076 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7077 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7078 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7079 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7080 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7081 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7082 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7083 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7084 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7085 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7086 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7087 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7088 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7089 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7090 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7091 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7092 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7093 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7094 ; GFX940-TGSPLIT-NEXT: s_endpgm
7095 ;
7096 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7097 ; GFX11-WGP: ; %bb.0: ; %entry
7098 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7099 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7100 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7101 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7102 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7103 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7104 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7105 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7106 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7107 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7108 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7109 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7110 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7111 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7112 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7113 ; GFX11-WGP-NEXT: buffer_gl0_inv
7114 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7115 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7116 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7117 ; GFX11-WGP-NEXT: s_endpgm
7118 ;
7119 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7120 ; GFX11-CU: ; %bb.0: ; %entry
7121 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7122 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7123 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7124 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7125 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7126 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7127 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7128 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7129 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7130 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7131 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7132 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7133 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7134 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7135 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7136 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7137 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7138 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7139 ; GFX11-CU-NEXT: s_endpgm
7140 ;
7141 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7142 ; GFX12-WGP: ; %bb.0: ; %entry
7143 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7144 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7145 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7146 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7147 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7148 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7149 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7150 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7151 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7152 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7153 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7154 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
7155 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7156 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7157 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
7158 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7159 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7160 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7161 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7162 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7163 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
7164 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7166 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7167 ; GFX12-WGP-NEXT: s_endpgm
7168 ;
7169 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
7170 ; GFX12-CU: ; %bb.0: ; %entry
7171 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7172 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7173 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7174 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7175 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7176 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7177 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7178 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7179 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7180 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7181 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7182 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7183 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
7184 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7185 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7186 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7187 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7188 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7189 ; GFX12-CU-NEXT: s_endpgm
7190 ptr %out, i32 %in, i32 %old) {
7191 entry:
7192 %gep = getelementptr i32, ptr %out, i32 4
7193 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
7194 %val0 = extractvalue { i32, i1 } %val, 0
7195 store i32 %val0, ptr %out, align 4
7196 ret void
7197 }
7199 define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
7200 ; GFX7-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7201 ; GFX7: ; %bb.0: ; %entry
7202 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7203 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7204 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7205 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7206 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7207 ; GFX7-NEXT: s_mov_b32 s6, s4
7208 ; GFX7-NEXT: s_mov_b32 s7, s5
7209 ; GFX7-NEXT: s_mov_b32 s11, s12
7210 ; GFX7-NEXT: s_mov_b32 s10, s13
7211 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7212 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7213 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7214 ; GFX7-NEXT: s_mov_b32 s7, s10
7215 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7216 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7217 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7218 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7219 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7220 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7221 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7222 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7223 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7224 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7225 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7226 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7227 ; GFX7-NEXT: s_endpgm
7228 ;
7229 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7230 ; GFX10-WGP: ; %bb.0: ; %entry
7231 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7232 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7233 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7234 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7235 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7236 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7237 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7238 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7239 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7240 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7241 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7242 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7243 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7244 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7245 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7246 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7247 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7248 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7249 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7250 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7251 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7252 ; GFX10-WGP-NEXT: buffer_gl0_inv
7253 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7254 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7255 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7256 ; GFX10-WGP-NEXT: s_endpgm
7257 ;
7258 ; GFX10-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7259 ; GFX10-CU: ; %bb.0: ; %entry
7260 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7261 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7262 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7263 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7264 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7265 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7266 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7267 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7268 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7269 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7270 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7271 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7272 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7273 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7274 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7275 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7276 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7277 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7278 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7279 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7280 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7281 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7282 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7283 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7284 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7285 ; GFX10-CU-NEXT: s_endpgm
7286 ;
7287 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7288 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7289 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7290 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7291 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7292 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7293 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7294 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7296 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7297 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7298 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7299 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7300 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7301 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7302 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7304 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7305 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7308 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7309 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7310 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7311 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7312 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7313 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7314 ; SKIP-CACHE-INV-NEXT: s_endpgm
7315 ;
7316 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7317 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7318 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7319 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7320 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7321 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7322 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7323 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7324 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7325 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7326 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7327 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7328 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7329 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7330 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7331 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7332 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7333 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7334 ;
7335 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7336 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7337 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7338 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7339 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7340 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7341 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7342 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7343 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7344 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7345 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7346 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7347 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7348 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7349 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7350 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7351 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7352 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7353 ;
7354 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7355 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7356 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7357 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7358 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7359 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7360 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7361 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7362 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7363 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7364 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7365 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7366 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7368 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7369 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7370 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7371 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7372 ;
7373 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7374 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7375 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7376 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7377 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7378 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7379 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7380 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7381 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7382 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7383 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7384 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7385 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7386 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7387 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7388 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7389 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7390 ; GFX940-TGSPLIT-NEXT: s_endpgm
7391 ;
7392 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7393 ; GFX11-WGP: ; %bb.0: ; %entry
7394 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7395 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7396 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7397 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7398 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7399 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7401 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7402 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7403 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7404 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7405 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7406 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7407 ; GFX11-WGP-NEXT: buffer_gl0_inv
7408 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7409 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7410 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7411 ; GFX11-WGP-NEXT: s_endpgm
7412 ;
7413 ; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7414 ; GFX11-CU: ; %bb.0: ; %entry
7415 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7416 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7417 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7418 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7419 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7420 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7421 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7422 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7423 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7424 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7425 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7426 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7427 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7428 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7429 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7430 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7431 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7432 ; GFX11-CU-NEXT: s_endpgm
7433 ;
7434 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7435 ; GFX12-WGP: ; %bb.0: ; %entry
7436 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7437 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7438 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7439 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7440 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7441 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7442 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7443 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7444 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7445 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7446 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7447 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7448 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7449 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7450 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7451 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
7452 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7453 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7454 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7455 ; GFX12-WGP-NEXT: s_endpgm
7456 ;
7457 ; GFX12-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
7458 ; GFX12-CU: ; %bb.0: ; %entry
7459 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7460 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7461 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7462 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7463 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7464 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7465 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7466 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7467 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7468 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7469 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7470 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
7471 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7472 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7473 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7474 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7475 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7476 ; GFX12-CU-NEXT: s_endpgm
7477 ptr %out, i32 %in, i32 %old) {
7478 entry:
7479 %gep = getelementptr i32, ptr %out, i32 4
7480 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
7481 %val0 = extractvalue { i32, i1 } %val, 0
7482 store i32 %val0, ptr %out, align 4
7483 ret void
7484 }
7486 define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
7487 ; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7488 ; GFX7: ; %bb.0: ; %entry
7489 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7490 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7491 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7492 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7493 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7494 ; GFX7-NEXT: s_mov_b32 s6, s4
7495 ; GFX7-NEXT: s_mov_b32 s7, s5
7496 ; GFX7-NEXT: s_mov_b32 s11, s12
7497 ; GFX7-NEXT: s_mov_b32 s10, s13
7498 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7499 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7500 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7501 ; GFX7-NEXT: s_mov_b32 s7, s10
7502 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7503 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7504 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7505 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7506 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7507 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7508 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7509 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7510 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7511 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7512 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7513 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7514 ; GFX7-NEXT: s_endpgm
7515 ;
7516 ; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7517 ; GFX10-WGP: ; %bb.0: ; %entry
7518 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7519 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7520 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7521 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7522 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7523 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7524 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7525 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7526 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7527 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7528 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7529 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7530 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7531 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7532 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7533 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7534 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7535 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7536 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7537 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7538 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7539 ; GFX10-WGP-NEXT: buffer_gl0_inv
7540 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7541 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7542 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7543 ; GFX10-WGP-NEXT: s_endpgm
7544 ;
7545 ; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7546 ; GFX10-CU: ; %bb.0: ; %entry
7547 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7548 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7549 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7550 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7551 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7552 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7553 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7554 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7555 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7556 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7557 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7558 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7559 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7560 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7561 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7562 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7563 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7564 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7565 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7566 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7567 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7568 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7569 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7570 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7571 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7572 ; GFX10-CU-NEXT: s_endpgm
7573 ;
7574 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7575 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7576 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7577 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7578 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7579 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7580 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7581 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7582 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7583 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7584 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7585 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7586 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7587 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7588 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7591 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7592 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7593 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7594 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7595 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7596 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7597 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7598 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7599 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7600 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7601 ; SKIP-CACHE-INV-NEXT: s_endpgm
7602 ;
7603 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7604 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7605 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7606 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7607 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7608 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7609 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7610 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7611 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7612 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7613 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7614 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7615 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7616 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7617 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7618 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7619 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7620 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7621 ;
7622 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7623 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7624 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7625 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7626 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7627 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7628 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7629 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7630 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7631 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7632 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7633 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7634 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7635 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7636 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7637 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7638 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7639 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7640 ;
7641 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7642 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7643 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7644 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7645 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7646 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7647 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7648 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7649 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7650 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7651 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7652 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7653 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7654 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7655 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7656 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7657 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7658 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7659 ;
7660 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7661 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7662 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7663 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7664 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7665 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7666 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7667 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7668 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7669 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7670 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7671 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7672 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7673 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7674 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7675 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7676 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7677 ; GFX940-TGSPLIT-NEXT: s_endpgm
7678 ;
7679 ; GFX11-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7680 ; GFX11-WGP: ; %bb.0: ; %entry
7681 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7682 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7683 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7684 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7685 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7686 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7687 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7688 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7689 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7690 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7691 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7692 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7693 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7694 ; GFX11-WGP-NEXT: buffer_gl0_inv
7695 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7696 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7697 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7698 ; GFX11-WGP-NEXT: s_endpgm
7699 ;
7700 ; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7701 ; GFX11-CU: ; %bb.0: ; %entry
7702 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7703 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7704 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7705 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7706 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7707 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
7708 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
7709 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7710 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
7711 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7712 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7713 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7714 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7715 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
7716 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
7717 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7718 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7719 ; GFX11-CU-NEXT: s_endpgm
7720 ;
7721 ; GFX12-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7722 ; GFX12-WGP: ; %bb.0: ; %entry
7723 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7724 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7725 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7726 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7727 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7728 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
7729 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
7730 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7731 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
7732 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7733 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7734 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7735 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7736 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7737 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7738 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
7739 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
7740 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
7741 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
7742 ; GFX12-WGP-NEXT: s_endpgm
7743 ;
7744 ; GFX12-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
7745 ; GFX12-CU: ; %bb.0: ; %entry
7746 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7747 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7748 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7749 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7750 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7751 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
7752 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
7753 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7754 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
7755 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7756 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7757 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
7758 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7759 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
7760 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
7761 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7762 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
7763 ; GFX12-CU-NEXT: s_endpgm
7764 ptr %out, i32 %in, i32 %old) {
7765 entry:
7766 %gep = getelementptr i32, ptr %out, i32 4
7767 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
7768 %val0 = extractvalue { i32, i1 } %val, 0
7769 store i32 %val0, ptr %out, align 4
7770 ret void
7771 }
7773 define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
7774 ; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7775 ; GFX7: ; %bb.0: ; %entry
7776 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7777 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7778 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7779 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7780 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7781 ; GFX7-NEXT: s_mov_b32 s6, s4
7782 ; GFX7-NEXT: s_mov_b32 s7, s5
7783 ; GFX7-NEXT: s_mov_b32 s11, s12
7784 ; GFX7-NEXT: s_mov_b32 s10, s13
7785 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7786 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7787 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7788 ; GFX7-NEXT: s_mov_b32 s7, s10
7789 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7790 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7791 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7792 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7793 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7794 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7795 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7796 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7797 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7798 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7799 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7800 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7801 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7802 ; GFX7-NEXT: s_endpgm
7803 ;
7804 ; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7805 ; GFX10-WGP: ; %bb.0: ; %entry
7806 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7807 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
7808 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
7809 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
7810 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7811 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
7812 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
7813 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
7814 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
7815 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
7816 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
7817 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7818 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
7819 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
7820 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
7821 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7822 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
7823 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
7824 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7825 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7826 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7827 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7828 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7829 ; GFX10-WGP-NEXT: buffer_gl0_inv
7830 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
7831 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
7832 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7833 ; GFX10-WGP-NEXT: s_endpgm
7834 ;
7835 ; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7836 ; GFX10-CU: ; %bb.0: ; %entry
7837 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7838 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
7839 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
7840 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
7841 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7842 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
7843 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
7844 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
7845 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
7846 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
7847 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
7848 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7849 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
7850 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
7851 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
7852 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7853 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
7854 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
7855 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7856 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7857 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7858 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7859 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
7860 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
7861 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7862 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7863 ; GFX10-CU-NEXT: s_endpgm
7864 ;
7865 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7866 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7867 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7868 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7869 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7870 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
7871 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7872 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
7873 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
7874 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
7875 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
7876 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
7877 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
7878 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
7879 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7880 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
7881 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
7882 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7883 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
7884 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7885 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7886 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7887 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7888 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7889 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7890 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7891 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7892 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7893 ; SKIP-CACHE-INV-NEXT: s_endpgm
7894 ;
7895 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7896 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7897 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7898 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7899 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7900 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7901 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7902 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7903 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7904 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7905 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7906 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7907 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7908 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7909 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7910 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7911 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7912 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7913 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7914 ;
7915 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7916 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7917 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7918 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7919 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7920 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7921 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7922 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7923 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
7924 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7925 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7926 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7927 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7928 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
7929 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7930 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7931 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
7932 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7933 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7934 ;
7935 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7936 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7937 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7938 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7939 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7940 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7941 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7942 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7943 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7944 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7945 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7946 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7947 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7948 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7949 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7950 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7951 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7952 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7953 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7954 ;
7955 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7956 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7957 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7958 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7959 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7960 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7961 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7962 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7963 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
7964 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7965 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
7966 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7967 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7968 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
7969 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7970 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7971 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7972 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7973 ; GFX940-TGSPLIT-NEXT: s_endpgm
7974 ;
7975 ; GFX11-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7976 ; GFX11-WGP: ; %bb.0: ; %entry
7977 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7978 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7979 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7980 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7981 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7982 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
7983 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
7984 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7985 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
7986 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7987 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7988 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7989 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7990 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
7991 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7992 ; GFX11-WGP-NEXT: buffer_gl0_inv
7993 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
7994 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
7995 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7996 ; GFX11-WGP-NEXT: s_endpgm
7997 ;
7998 ; GFX11-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
7999 ; GFX11-CU: ; %bb.0: ; %entry
8000 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8001 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8002 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8003 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8004 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8005 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8006 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8007 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8008 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8009 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8010 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8011 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8012 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8013 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8014 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8015 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8016 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8017 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8018 ; GFX11-CU-NEXT: s_endpgm
8019 ;
8020 ; GFX12-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
8021 ; GFX12-WGP: ; %bb.0: ; %entry
8022 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8023 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8024 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8025 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8026 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8027 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8028 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8029 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8030 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8031 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8032 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8033 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8034 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8035 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8036 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8037 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8038 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8039 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8040 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8041 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8042 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8043 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8044 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8045 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8046 ; GFX12-WGP-NEXT: s_endpgm
8047 ;
8048 ; GFX12-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
8049 ; GFX12-CU: ; %bb.0: ; %entry
8050 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8051 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8052 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8053 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8054 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8055 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8056 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8057 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8058 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8059 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8060 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8061 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8062 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
8063 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8064 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8065 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8066 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8067 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8068 ; GFX12-CU-NEXT: s_endpgm
8069 ptr %out, i32 %in, i32 %old) {
8070 entry:
8071 %gep = getelementptr i32, ptr %out, i32 4
8072 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
8073 %val0 = extractvalue { i32, i1 } %val, 0
8074 store i32 %val0, ptr %out, align 4
8075 ret void
8076 }
8078 define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
8079 ; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8080 ; GFX7: ; %bb.0: ; %entry
8081 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8082 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8083 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8084 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8085 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8086 ; GFX7-NEXT: s_mov_b32 s6, s4
8087 ; GFX7-NEXT: s_mov_b32 s7, s5
8088 ; GFX7-NEXT: s_mov_b32 s11, s12
8089 ; GFX7-NEXT: s_mov_b32 s10, s13
8090 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8091 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8092 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8093 ; GFX7-NEXT: s_mov_b32 s7, s10
8094 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8095 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8096 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8097 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8098 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8099 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8100 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8101 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8102 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8103 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8104 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8105 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8106 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8107 ; GFX7-NEXT: s_endpgm
8108 ;
8109 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8110 ; GFX10-WGP: ; %bb.0: ; %entry
8111 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8112 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8113 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8114 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8115 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8116 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8117 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8118 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8119 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8120 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8121 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8122 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8123 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8124 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8125 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8126 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8127 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8128 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8129 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8130 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8131 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8132 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8133 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8134 ; GFX10-WGP-NEXT: buffer_gl0_inv
8135 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8136 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8137 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8138 ; GFX10-WGP-NEXT: s_endpgm
8139 ;
8140 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8141 ; GFX10-CU: ; %bb.0: ; %entry
8142 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8143 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8144 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8145 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8146 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8147 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8148 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8149 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8150 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8151 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8152 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8153 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8154 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8155 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8156 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8157 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8158 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8159 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8160 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8161 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8162 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8163 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8164 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8165 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8166 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8167 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8168 ; GFX10-CU-NEXT: s_endpgm
8169 ;
8170 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8171 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8172 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8173 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8174 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8175 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8176 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8177 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8178 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8179 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8180 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8181 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8182 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8183 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8184 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8185 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8186 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8187 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8188 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8189 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8190 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8191 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8192 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8193 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8194 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8195 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8196 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8197 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8198 ; SKIP-CACHE-INV-NEXT: s_endpgm
8199 ;
8200 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8201 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8202 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8203 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8204 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8205 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8206 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8207 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8208 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8209 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8210 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8211 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8212 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8213 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8214 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8215 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8216 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8217 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8218 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8219 ;
8220 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8221 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8222 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8223 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8224 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8225 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8226 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8227 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8228 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8229 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8230 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8231 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8232 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8233 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8234 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8235 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8236 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8237 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8238 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8239 ;
8240 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8241 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8242 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8243 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8244 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8245 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8246 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8247 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8248 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8249 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8250 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8251 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8252 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8253 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8254 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8255 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8256 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8257 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8258 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8259 ;
8260 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8261 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8262 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8263 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8264 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8265 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8266 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8267 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8268 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8269 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8270 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8271 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8272 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8273 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8274 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8275 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8276 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8277 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8278 ; GFX940-TGSPLIT-NEXT: s_endpgm
8279 ;
8280 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8281 ; GFX11-WGP: ; %bb.0: ; %entry
8282 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8283 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8284 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8285 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8286 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8287 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8288 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8289 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8290 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8291 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8292 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8293 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8294 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8295 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8296 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8297 ; GFX11-WGP-NEXT: buffer_gl0_inv
8298 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8299 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8300 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8301 ; GFX11-WGP-NEXT: s_endpgm
8302 ;
8303 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8304 ; GFX11-CU: ; %bb.0: ; %entry
8305 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8306 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8307 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8308 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8309 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8310 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8311 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8312 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8313 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8314 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8315 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8316 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8317 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8318 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8319 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8320 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8321 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8322 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8323 ; GFX11-CU-NEXT: s_endpgm
8324 ;
8325 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8326 ; GFX12-WGP: ; %bb.0: ; %entry
8327 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8328 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8329 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8330 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8331 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8332 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8333 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8334 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8335 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8336 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8337 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8338 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8339 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8340 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8341 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8342 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8343 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8344 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8345 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8346 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8347 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8348 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8349 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8350 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8351 ; GFX12-WGP-NEXT: s_endpgm
8352 ;
8353 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
8354 ; GFX12-CU: ; %bb.0: ; %entry
8355 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8356 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8357 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8358 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8359 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8360 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8361 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8362 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8363 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8364 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8365 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8366 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8367 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
8368 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8369 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8370 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8371 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8372 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8373 ; GFX12-CU-NEXT: s_endpgm
8374 ptr %out, i32 %in, i32 %old) {
8375 entry:
8376 %gep = getelementptr i32, ptr %out, i32 4
8377 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
8378 %val0 = extractvalue { i32, i1 } %val, 0
8379 store i32 %val0, ptr %out, align 4
8380 ret void
8381 }
8383 define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
8384 ; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8385 ; GFX7: ; %bb.0: ; %entry
8386 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8387 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8388 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8389 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8390 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8391 ; GFX7-NEXT: s_mov_b32 s6, s4
8392 ; GFX7-NEXT: s_mov_b32 s7, s5
8393 ; GFX7-NEXT: s_mov_b32 s11, s12
8394 ; GFX7-NEXT: s_mov_b32 s10, s13
8395 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8396 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8397 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8398 ; GFX7-NEXT: s_mov_b32 s7, s10
8399 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8400 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8401 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8402 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8403 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8404 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8405 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8406 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8407 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8408 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8409 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8410 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8411 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8412 ; GFX7-NEXT: s_endpgm
8413 ;
8414 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8415 ; GFX10-WGP: ; %bb.0: ; %entry
8416 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8417 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8418 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8419 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8420 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8421 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8422 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8423 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8424 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8425 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8426 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8427 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8428 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8429 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8430 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8431 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8432 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8433 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8434 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8435 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8436 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8437 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8438 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8439 ; GFX10-WGP-NEXT: buffer_gl0_inv
8440 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8441 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8442 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8443 ; GFX10-WGP-NEXT: s_endpgm
8444 ;
8445 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8446 ; GFX10-CU: ; %bb.0: ; %entry
8447 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8448 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8449 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8450 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8451 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8452 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8453 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8454 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8455 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8456 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8457 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8458 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8459 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8460 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8461 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8462 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8463 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8464 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8465 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8466 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8467 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8468 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8469 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8470 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8471 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8472 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8473 ; GFX10-CU-NEXT: s_endpgm
8474 ;
8475 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8476 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8477 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8478 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8479 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8480 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8481 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8482 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8483 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8484 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8485 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8486 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8487 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8488 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8489 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8490 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8491 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8492 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8493 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8494 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8495 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8496 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8497 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8498 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8499 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8500 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8501 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8502 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8503 ; SKIP-CACHE-INV-NEXT: s_endpgm
8504 ;
8505 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8506 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8507 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8508 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8509 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8510 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8511 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8512 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8513 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8514 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8515 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8516 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8517 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8518 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8519 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8520 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8521 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8522 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8523 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8524 ;
8525 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8526 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8527 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8528 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8529 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8530 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8531 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8532 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8533 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8534 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8535 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8536 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8537 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8538 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8539 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8540 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8541 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8542 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8543 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8544 ;
8545 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8546 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8547 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8548 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8549 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8550 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8551 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8552 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8553 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8554 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8555 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8556 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8557 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8558 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8559 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8560 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8561 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8562 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8563 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8564 ;
8565 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8566 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8567 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8568 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8569 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8570 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8571 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8572 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8573 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8574 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8575 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8576 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8577 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8578 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8579 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8580 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8581 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8582 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8583 ; GFX940-TGSPLIT-NEXT: s_endpgm
8584 ;
8585 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8586 ; GFX11-WGP: ; %bb.0: ; %entry
8587 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8588 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8589 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8590 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8591 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8592 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8593 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8594 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8595 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8596 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8597 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8598 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8599 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8600 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8601 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8602 ; GFX11-WGP-NEXT: buffer_gl0_inv
8603 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8604 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8605 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8606 ; GFX11-WGP-NEXT: s_endpgm
8607 ;
8608 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8609 ; GFX11-CU: ; %bb.0: ; %entry
8610 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8611 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8612 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8613 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8614 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8615 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8616 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8617 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8618 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8619 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8620 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8621 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8622 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8623 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8624 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8625 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8626 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8627 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8628 ; GFX11-CU-NEXT: s_endpgm
8629 ;
8630 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8631 ; GFX12-WGP: ; %bb.0: ; %entry
8632 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8633 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8634 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8635 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8636 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8637 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8638 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8639 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8640 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8641 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8642 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8643 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8644 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8645 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8646 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8647 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8648 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8649 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8650 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8651 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8652 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8653 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8654 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8655 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8656 ; GFX12-WGP-NEXT: s_endpgm
8657 ;
8658 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
8659 ; GFX12-CU: ; %bb.0: ; %entry
8660 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8661 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8662 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8663 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8664 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8665 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8666 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8667 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8668 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8669 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8670 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8671 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8672 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
8673 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8674 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8675 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8676 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8677 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8678 ; GFX12-CU-NEXT: s_endpgm
8679 ptr %out, i32 %in, i32 %old) {
8680 entry:
8681 %gep = getelementptr i32, ptr %out, i32 4
8682 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
8683 %val0 = extractvalue { i32, i1 } %val, 0
8684 store i32 %val0, ptr %out, align 4
8685 ret void
8686 }
8688 define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
8689 ; GFX7-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8690 ; GFX7: ; %bb.0: ; %entry
8691 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8692 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8693 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8694 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8695 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8696 ; GFX7-NEXT: s_mov_b32 s6, s4
8697 ; GFX7-NEXT: s_mov_b32 s7, s5
8698 ; GFX7-NEXT: s_mov_b32 s11, s12
8699 ; GFX7-NEXT: s_mov_b32 s10, s13
8700 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8701 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8702 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8703 ; GFX7-NEXT: s_mov_b32 s7, s10
8704 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8705 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8706 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8707 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8708 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8709 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8710 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8711 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8712 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8713 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8714 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8715 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8716 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8717 ; GFX7-NEXT: s_endpgm
8718 ;
8719 ; GFX10-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8720 ; GFX10-WGP: ; %bb.0: ; %entry
8721 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8722 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
8723 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
8724 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
8725 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8726 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
8727 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
8728 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
8729 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
8730 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
8731 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
8732 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8733 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
8734 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
8735 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
8736 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8737 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
8738 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
8739 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8740 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8741 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8742 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8743 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8744 ; GFX10-WGP-NEXT: buffer_gl0_inv
8745 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
8746 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
8747 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
8748 ; GFX10-WGP-NEXT: s_endpgm
8749 ;
8750 ; GFX10-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8751 ; GFX10-CU: ; %bb.0: ; %entry
8752 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8753 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
8754 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
8755 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
8756 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8757 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
8758 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
8759 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
8760 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
8761 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
8762 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
8763 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8764 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
8765 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
8766 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
8767 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8768 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
8769 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
8770 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8771 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8772 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8773 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8774 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
8775 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
8776 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8777 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
8778 ; GFX10-CU-NEXT: s_endpgm
8779 ;
8780 ; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8781 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8782 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8783 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8784 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8785 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
8786 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8787 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
8788 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
8789 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
8790 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
8791 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
8792 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
8793 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
8794 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8795 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
8796 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
8797 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8798 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
8799 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
8800 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
8801 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8802 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8803 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8804 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8805 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8806 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8807 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
8808 ; SKIP-CACHE-INV-NEXT: s_endpgm
8809 ;
8810 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8811 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8812 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8813 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8814 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8815 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8816 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8817 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8818 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8819 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8820 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8821 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8822 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8823 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8824 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8825 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8826 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8827 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
8828 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8829 ;
8830 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8831 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8832 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8833 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8834 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8835 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8836 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8837 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8838 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
8839 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8840 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8841 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8842 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8843 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
8844 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8845 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8846 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
8847 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
8848 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8849 ;
8850 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8851 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8852 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8853 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8854 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8855 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8856 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8857 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8858 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8859 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8860 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8861 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8862 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8863 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8864 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8865 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8866 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8867 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8868 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8869 ;
8870 ; GFX940-TGSPLIT-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8871 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8872 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8873 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8874 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8875 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8876 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8877 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8878 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
8879 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8880 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
8881 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8882 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8883 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
8884 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8885 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8886 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8887 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
8888 ; GFX940-TGSPLIT-NEXT: s_endpgm
8889 ;
8890 ; GFX11-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8891 ; GFX11-WGP: ; %bb.0: ; %entry
8892 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8893 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8894 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8895 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8896 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8897 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
8898 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
8899 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8900 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
8901 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8902 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8903 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8904 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8905 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8906 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8907 ; GFX11-WGP-NEXT: buffer_gl0_inv
8908 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
8909 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
8910 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
8911 ; GFX11-WGP-NEXT: s_endpgm
8912 ;
8913 ; GFX11-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8914 ; GFX11-CU: ; %bb.0: ; %entry
8915 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8916 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8917 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8918 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8919 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8920 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
8921 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
8922 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8923 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
8924 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8925 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8926 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8927 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
8928 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8929 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
8930 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
8931 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8932 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
8933 ; GFX11-CU-NEXT: s_endpgm
8934 ;
8935 ; GFX12-WGP-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8936 ; GFX12-WGP: ; %bb.0: ; %entry
8937 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8938 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8939 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8940 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8941 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8942 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
8943 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
8944 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8945 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
8946 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8947 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8948 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8949 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8950 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8951 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8952 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8953 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8954 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8955 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8956 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8957 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8958 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
8959 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
8960 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
8961 ; GFX12-WGP-NEXT: s_endpgm
8962 ;
8963 ; GFX12-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
8964 ; GFX12-CU: ; %bb.0: ; %entry
8965 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8966 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8967 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8968 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8969 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8970 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
8971 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
8972 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8973 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
8974 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8975 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8976 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8977 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
8978 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8979 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
8980 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
8981 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8982 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
8983 ; GFX12-CU-NEXT: s_endpgm
8984 ptr %out, i32 %in, i32 %old) {
8985 entry:
8986 %gep = getelementptr i32, ptr %out, i32 4
8987 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
8988 %val0 = extractvalue { i32, i1 } %val, 0
8989 store i32 %val0, ptr %out, align 4
8990 ret void
8991 }
8993 define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
8994 ; GFX7-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
8995 ; GFX7: ; %bb.0: ; %entry
8996 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8997 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8998 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8999 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9000 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9001 ; GFX7-NEXT: s_mov_b32 s6, s4
9002 ; GFX7-NEXT: s_mov_b32 s7, s5
9003 ; GFX7-NEXT: s_mov_b32 s11, s12
9004 ; GFX7-NEXT: s_mov_b32 s10, s13
9005 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9006 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9007 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9008 ; GFX7-NEXT: s_mov_b32 s7, s10
9009 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9010 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9011 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9012 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9013 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9014 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9015 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9016 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9017 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9018 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9019 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9020 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9021 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9022 ; GFX7-NEXT: s_endpgm
9023 ;
9024 ; GFX10-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9025 ; GFX10-WGP: ; %bb.0: ; %entry
9026 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9027 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9028 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9029 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9030 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9031 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9032 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9033 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9034 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9035 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9036 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9037 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9038 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9039 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9040 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9041 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9042 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9043 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9044 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9045 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9046 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9047 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9048 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9049 ; GFX10-WGP-NEXT: buffer_gl0_inv
9050 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9051 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9052 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9053 ; GFX10-WGP-NEXT: s_endpgm
9054 ;
9055 ; GFX10-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9056 ; GFX10-CU: ; %bb.0: ; %entry
9057 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9058 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9059 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9060 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9061 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9062 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9063 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9064 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9065 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9066 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9067 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9068 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9069 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9070 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9071 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9072 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9073 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9074 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9075 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9076 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9077 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9078 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9079 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9080 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9081 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9082 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9083 ; GFX10-CU-NEXT: s_endpgm
9084 ;
9085 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9086 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9087 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9088 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9089 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9090 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9091 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9092 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9093 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9094 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9095 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9096 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9097 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9098 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9099 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9100 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9101 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9102 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9103 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9104 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9105 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9106 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9107 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9108 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9109 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9110 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9111 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9112 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9113 ; SKIP-CACHE-INV-NEXT: s_endpgm
9114 ;
9115 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9116 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9117 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9118 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9119 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9120 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9121 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9122 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9123 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9124 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9125 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9126 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9127 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9128 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9129 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9130 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9131 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9132 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9133 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9134 ;
9135 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9136 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9137 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9138 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9139 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9140 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9141 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9142 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9143 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9144 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9145 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9146 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9147 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9148 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9149 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9150 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9151 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9152 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9153 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9154 ;
9155 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9156 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9157 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9158 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9159 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9160 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9161 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9162 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9164 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9165 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9166 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9167 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9168 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9169 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9170 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9171 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9172 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9173 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9174 ;
9175 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9176 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9177 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9178 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9179 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9180 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9181 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9182 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9183 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9184 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9185 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9186 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9187 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9188 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9189 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9190 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9191 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9192 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9193 ; GFX940-TGSPLIT-NEXT: s_endpgm
9194 ;
9195 ; GFX11-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9196 ; GFX11-WGP: ; %bb.0: ; %entry
9197 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9198 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9199 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9200 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9201 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9202 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9203 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9204 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9205 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9206 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9207 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9208 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9209 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9210 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9211 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9212 ; GFX11-WGP-NEXT: buffer_gl0_inv
9213 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9214 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9215 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9216 ; GFX11-WGP-NEXT: s_endpgm
9217 ;
9218 ; GFX11-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9219 ; GFX11-CU: ; %bb.0: ; %entry
9220 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9221 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9222 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9223 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9224 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9225 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9226 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9227 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9228 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9229 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9230 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9231 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9232 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9233 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9234 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9235 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9236 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9237 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9238 ; GFX11-CU-NEXT: s_endpgm
9239 ;
9240 ; GFX12-WGP-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9241 ; GFX12-WGP: ; %bb.0: ; %entry
9242 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9243 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9244 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9245 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9246 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9247 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9248 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9249 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9250 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9251 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9252 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9253 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9254 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9255 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9256 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9257 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9258 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9259 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9260 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9261 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9262 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9263 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9264 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9265 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9266 ; GFX12-WGP-NEXT: s_endpgm
9267 ;
9268 ; GFX12-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
9269 ; GFX12-CU: ; %bb.0: ; %entry
9270 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9271 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9272 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9273 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9274 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9275 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9276 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9277 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9278 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9279 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9280 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9281 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9282 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
9283 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9284 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9285 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9286 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9287 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9288 ; GFX12-CU-NEXT: s_endpgm
9289 ptr %out, i32 %in, i32 %old) {
9290 entry:
9291 %gep = getelementptr i32, ptr %out, i32 4
9292 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
9293 %val0 = extractvalue { i32, i1 } %val, 0
9294 store i32 %val0, ptr %out, align 4
9295 ret void
9296 }
9298 define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
9299 ; GFX7-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9300 ; GFX7: ; %bb.0: ; %entry
9301 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9302 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9303 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9304 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9305 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9306 ; GFX7-NEXT: s_mov_b32 s6, s4
9307 ; GFX7-NEXT: s_mov_b32 s7, s5
9308 ; GFX7-NEXT: s_mov_b32 s11, s12
9309 ; GFX7-NEXT: s_mov_b32 s10, s13
9310 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9311 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9312 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9313 ; GFX7-NEXT: s_mov_b32 s7, s10
9314 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9315 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9316 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9317 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9318 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9319 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9320 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9321 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9322 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9323 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9324 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9325 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9326 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9327 ; GFX7-NEXT: s_endpgm
9328 ;
9329 ; GFX10-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9330 ; GFX10-WGP: ; %bb.0: ; %entry
9331 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9332 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9333 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9334 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9335 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9336 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9337 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9338 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9339 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9340 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9341 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9342 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9343 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9344 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9345 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9346 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9347 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9348 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9349 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9350 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9351 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9352 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9353 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9354 ; GFX10-WGP-NEXT: buffer_gl0_inv
9355 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9356 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9357 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9358 ; GFX10-WGP-NEXT: s_endpgm
9359 ;
9360 ; GFX10-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9361 ; GFX10-CU: ; %bb.0: ; %entry
9362 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9363 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9364 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9365 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9366 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9367 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9368 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9369 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9370 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9371 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9372 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9373 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9374 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9375 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9376 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9377 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9378 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9379 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9380 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9381 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9382 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9383 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9384 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9385 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9386 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9387 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9388 ; GFX10-CU-NEXT: s_endpgm
9389 ;
9390 ; SKIP-CACHE-INV-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9391 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9392 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9393 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9394 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9395 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9396 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9397 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9398 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9399 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9400 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9401 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9402 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9403 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9404 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9405 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9406 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9407 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9408 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9409 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9410 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9411 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9412 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9413 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9414 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9415 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9416 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9417 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9418 ; SKIP-CACHE-INV-NEXT: s_endpgm
9419 ;
9420 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9421 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9422 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9423 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9424 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9425 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9426 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9427 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9428 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9429 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9430 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9431 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9432 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9433 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9434 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9435 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9436 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9437 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9438 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9439 ;
9440 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9441 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9442 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9443 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9444 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9445 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9446 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9447 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9448 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9449 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9450 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9451 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9452 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9453 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9454 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9455 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9456 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9457 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9458 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9459 ;
9460 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9461 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9462 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9463 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9464 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9465 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9466 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9467 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9468 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9469 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9470 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9471 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9472 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9473 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9474 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9475 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9476 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9477 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9478 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9479 ;
9480 ; GFX940-TGSPLIT-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9481 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9482 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9483 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9484 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9485 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9486 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9487 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9488 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9489 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9490 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9491 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9492 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9493 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9494 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9495 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9496 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9497 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9498 ; GFX940-TGSPLIT-NEXT: s_endpgm
9499 ;
9500 ; GFX11-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9501 ; GFX11-WGP: ; %bb.0: ; %entry
9502 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9503 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9504 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9505 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9506 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9507 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9508 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9509 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9510 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9511 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9512 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9513 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9514 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9515 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9516 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9517 ; GFX11-WGP-NEXT: buffer_gl0_inv
9518 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9519 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9520 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9521 ; GFX11-WGP-NEXT: s_endpgm
9522 ;
9523 ; GFX11-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9524 ; GFX11-CU: ; %bb.0: ; %entry
9525 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9526 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9527 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9528 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9529 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9530 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9531 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9532 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9533 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9534 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9535 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9536 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9537 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9538 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9539 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9540 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9541 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9542 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9543 ; GFX11-CU-NEXT: s_endpgm
9544 ;
9545 ; GFX12-WGP-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9546 ; GFX12-WGP: ; %bb.0: ; %entry
9547 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9548 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9549 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9550 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9551 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9552 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9553 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9554 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9555 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9556 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9557 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9558 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9559 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9560 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9561 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9562 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9563 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9564 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9565 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9566 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9567 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9568 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9569 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9570 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9571 ; GFX12-WGP-NEXT: s_endpgm
9572 ;
9573 ; GFX12-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
9574 ; GFX12-CU: ; %bb.0: ; %entry
9575 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9576 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9577 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9578 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9579 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9580 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9581 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9582 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9583 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9584 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9585 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9586 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9587 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
9588 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9589 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9590 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9591 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9592 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9593 ; GFX12-CU-NEXT: s_endpgm
9594 ptr %out, i32 %in, i32 %old) {
9595 entry:
9596 %gep = getelementptr i32, ptr %out, i32 4
9597 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
9598 %val0 = extractvalue { i32, i1 } %val, 0
9599 store i32 %val0, ptr %out, align 4
9600 ret void
9601 }
9603 define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
9604 ; GFX7-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9605 ; GFX7: ; %bb.0: ; %entry
9606 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9607 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9608 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9609 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9610 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9611 ; GFX7-NEXT: s_mov_b32 s6, s4
9612 ; GFX7-NEXT: s_mov_b32 s7, s5
9613 ; GFX7-NEXT: s_mov_b32 s11, s12
9614 ; GFX7-NEXT: s_mov_b32 s10, s13
9615 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9616 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9617 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9618 ; GFX7-NEXT: s_mov_b32 s7, s10
9619 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9620 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9621 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9622 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9623 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9624 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9625 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9626 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9627 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9628 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9629 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9630 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9631 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9632 ; GFX7-NEXT: s_endpgm
9633 ;
9634 ; GFX10-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9635 ; GFX10-WGP: ; %bb.0: ; %entry
9636 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9637 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9638 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9639 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9640 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9641 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9642 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9643 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9644 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9645 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9646 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9647 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9648 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9649 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9650 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9651 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9652 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9653 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9654 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9655 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9656 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9657 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9658 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9659 ; GFX10-WGP-NEXT: buffer_gl0_inv
9660 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9661 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9662 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9663 ; GFX10-WGP-NEXT: s_endpgm
9664 ;
9665 ; GFX10-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9666 ; GFX10-CU: ; %bb.0: ; %entry
9667 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9668 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9669 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9670 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9671 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9672 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9673 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9674 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9675 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9676 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9677 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9678 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9679 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9680 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9681 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9682 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9683 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9684 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9685 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9686 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9687 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9688 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9689 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9690 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9691 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9692 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9693 ; GFX10-CU-NEXT: s_endpgm
9694 ;
9695 ; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9696 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9697 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9698 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9699 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9700 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
9701 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9702 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
9703 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
9704 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
9705 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
9706 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
9707 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
9708 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
9709 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9710 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
9711 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9712 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9713 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
9714 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
9715 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
9716 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9717 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9718 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9719 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9720 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9721 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9722 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9723 ; SKIP-CACHE-INV-NEXT: s_endpgm
9724 ;
9725 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9726 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9727 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9728 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9729 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9730 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9731 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9732 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9733 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9734 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9735 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9736 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9737 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9738 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9739 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9740 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9741 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9742 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9743 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9744 ;
9745 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9746 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9747 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9748 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9749 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9750 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9751 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9752 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9753 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
9754 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9755 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9756 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9757 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9758 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9759 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9760 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9761 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
9762 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9763 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9764 ;
9765 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9766 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9767 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9768 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9769 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9770 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9771 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9772 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9773 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9774 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9775 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9776 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9777 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9778 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9779 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9780 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9781 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9782 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9783 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9784 ;
9785 ; GFX940-TGSPLIT-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9786 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9787 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9788 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9789 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9790 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9791 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9792 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9793 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
9794 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9795 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
9796 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9797 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9798 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9799 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9800 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9801 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9802 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9803 ; GFX940-TGSPLIT-NEXT: s_endpgm
9804 ;
9805 ; GFX11-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9806 ; GFX11-WGP: ; %bb.0: ; %entry
9807 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9808 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9809 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9810 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9811 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9812 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
9813 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
9814 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9815 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
9816 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9817 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9818 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9819 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9820 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9821 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9822 ; GFX11-WGP-NEXT: buffer_gl0_inv
9823 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
9824 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
9825 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9826 ; GFX11-WGP-NEXT: s_endpgm
9827 ;
9828 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9829 ; GFX11-CU: ; %bb.0: ; %entry
9830 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9831 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9832 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9833 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9834 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9835 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
9836 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
9837 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9838 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
9839 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9840 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9841 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9842 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9843 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9844 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
9845 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
9846 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9847 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9848 ; GFX11-CU-NEXT: s_endpgm
9849 ;
9850 ; GFX12-WGP-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9851 ; GFX12-WGP: ; %bb.0: ; %entry
9852 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9853 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9854 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9855 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9856 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9857 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
9858 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
9859 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9860 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
9861 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9862 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9863 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9864 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9865 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9866 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9867 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9868 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9869 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9870 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9871 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9872 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9873 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
9874 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
9875 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
9876 ; GFX12-WGP-NEXT: s_endpgm
9877 ;
9878 ; GFX12-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
9879 ; GFX12-CU: ; %bb.0: ; %entry
9880 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9881 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9882 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9883 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9884 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9885 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
9886 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
9887 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9888 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
9889 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9890 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9891 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9892 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
9893 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9894 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
9895 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
9896 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9897 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
9898 ; GFX12-CU-NEXT: s_endpgm
9899 ptr %out, i32 %in, i32 %old) {
9900 entry:
9901 %gep = getelementptr i32, ptr %out, i32 4
9902 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
9903 %val0 = extractvalue { i32, i1 } %val, 0
9904 store i32 %val0, ptr %out, align 4
9905 ret void
9906 }
9908 define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
9909 ; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
9910 ; GFX7: ; %bb.0: ; %entry
9911 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9912 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9913 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9914 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9915 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9916 ; GFX7-NEXT: s_mov_b32 s6, s4
9917 ; GFX7-NEXT: s_mov_b32 s7, s5
9918 ; GFX7-NEXT: s_mov_b32 s11, s12
9919 ; GFX7-NEXT: s_mov_b32 s10, s13
9920 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9921 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9922 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9923 ; GFX7-NEXT: s_mov_b32 s7, s10
9924 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9925 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9926 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9927 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9928 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9929 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9930 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9931 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9932 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9933 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9934 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9935 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9936 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9937 ; GFX7-NEXT: s_endpgm
9938 ;
9939 ; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
9940 ; GFX10-WGP: ; %bb.0: ; %entry
9941 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9942 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
9943 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
9944 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
9945 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9946 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
9947 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
9948 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
9949 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
9950 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
9951 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
9952 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9953 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
9954 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
9955 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
9956 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9957 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
9958 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
9959 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9960 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9961 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9962 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9963 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9964 ; GFX10-WGP-NEXT: buffer_gl0_inv
9965 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9966 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9967 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9968 ; GFX10-WGP-NEXT: s_endpgm
9969 ;
9970 ; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
9971 ; GFX10-CU: ; %bb.0: ; %entry
9972 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9973 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
9974 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
9975 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
9976 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9977 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
9978 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
9979 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
9980 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
9981 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
9982 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
9983 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9984 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
9985 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
9986 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
9987 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9988 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
9989 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
9990 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9991 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9992 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9993 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9994 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9995 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9996 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9997 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9998 ; GFX10-CU-NEXT: s_endpgm
9999 ;
10000 ; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10001 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10002 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10003 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10004 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10005 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
10006 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10007 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
10008 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
10009 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
10010 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
10011 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
10012 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
10013 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
10014 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10015 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
10016 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10017 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10018 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
10019 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10020 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10021 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10022 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10023 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10024 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10025 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10026 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
10027 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10028 ; SKIP-CACHE-INV-NEXT: s_endpgm
10029 ;
10030 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10031 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10032 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10033 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10034 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10035 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10036 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10037 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10038 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10039 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10040 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10041 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10042 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10043 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10044 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10045 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10046 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10047 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10048 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10049 ;
10050 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10051 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10052 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10053 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10054 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10055 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10056 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10057 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10058 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
10059 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10060 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10061 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10062 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10063 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10064 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10065 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10066 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10067 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10068 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10069 ;
10070 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10071 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10072 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10073 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10074 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10075 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10076 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10077 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10078 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10079 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10080 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10081 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10082 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10083 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10084 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10085 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10086 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10087 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10088 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10089 ;
10090 ; GFX940-TGSPLIT-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10091 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10092 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10093 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10094 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10095 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10096 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10097 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10098 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
10099 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10100 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
10101 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10102 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10103 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10104 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10105 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10106 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10107 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10108 ; GFX940-TGSPLIT-NEXT: s_endpgm
10109 ;
10110 ; GFX11-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10111 ; GFX11-WGP: ; %bb.0: ; %entry
10112 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10113 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10114 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10115 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10116 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10117 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
10118 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10119 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10120 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
10121 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10122 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10123 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10124 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10125 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10126 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10127 ; GFX11-WGP-NEXT: buffer_gl0_inv
10128 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10129 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10130 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10131 ; GFX11-WGP-NEXT: s_endpgm
10132 ;
10133 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10134 ; GFX11-CU: ; %bb.0: ; %entry
10135 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10136 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10137 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10138 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10139 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10140 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
10141 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10142 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10143 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
10144 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10145 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10146 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10147 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10148 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10149 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10150 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10151 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
10152 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10153 ; GFX11-CU-NEXT: s_endpgm
10154 ;
10155 ; GFX12-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10156 ; GFX12-WGP: ; %bb.0: ; %entry
10157 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10158 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10159 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10160 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10161 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10162 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
10163 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10164 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
10166 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10167 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10168 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
10169 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10170 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10171 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10172 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10173 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
10174 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10175 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10176 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10177 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10178 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10179 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10180 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10181 ; GFX12-WGP-NEXT: s_endpgm
10182 ;
10183 ; GFX12-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10184 ; GFX12-CU: ; %bb.0: ; %entry
10185 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10186 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10187 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10188 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10189 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10190 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
10191 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10192 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10193 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
10194 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10195 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10196 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
10197 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
10198 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
10199 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10200 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10201 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
10202 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10203 ; GFX12-CU-NEXT: s_endpgm
10204 ptr %out, i32 %in, i32 %old) {
10205 entry:
10206 %gep = getelementptr i32, ptr %out, i32 4
10207 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
10208 %val0 = extractvalue { i32, i1 } %val, 0
10209 store i32 %val0, ptr %out, align 4
10210 ret void
10211 }
10213 define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
10214 ; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
10215 ; GFX7: ; %bb.0: ; %entry
10216 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
10217 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10218 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
10219 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10220 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10221 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10222 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
10223 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10224 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10225 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10226 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10227 ; GFX7-NEXT: s_endpgm
10228 ;
10229 ; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
10230 ; GFX10-WGP: ; %bb.0: ; %entry
10231 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
10232 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10233 ; GFX10-WGP-NEXT: s_nop 0
10234 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10235 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10236 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10238 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
10239 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10240 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10241 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10242 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10243 ; GFX10-WGP-NEXT: s_endpgm
10244 ;
10245 ; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
10246 ; GFX10-CU: ; %bb.0: ; %entry
10247 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
10248 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10249 ; GFX10-CU-NEXT: s_nop 0
10250 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10251 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10252 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10253 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10254 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
10255 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10256 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10257 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10258 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10259 ; GFX10-CU-NEXT: s_endpgm
10260 ;
10261 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
10262 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10263 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
10264 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10265 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
10266 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10267 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10268 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10269 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
10270 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10271 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10272 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10273 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10274 ; SKIP-CACHE-INV-NEXT: s_endpgm
10275 ;
10276 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
10277 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10278 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10279 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10280 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
10281 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10282 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10283 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10284 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10285 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10286 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10287 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10288 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10289 ;
10290 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
10291 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10292 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10293 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10294 ; GFX90A-TGSPLIT-NEXT: s_nop 0
10295 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10296 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10297 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10298 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10299 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10300 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10301 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10302 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10303 ;
10304 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
10305 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10306 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10307 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10308 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
10309 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10310 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10311 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10312 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10313 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10314 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10315 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10316 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10317 ;
10318 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_load:
10319 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10320 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10321 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10322 ; GFX940-TGSPLIT-NEXT: s_nop 0
10323 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10324 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10325 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10326 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10327 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10328 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10329 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10330 ; GFX940-TGSPLIT-NEXT: s_endpgm
10331 ;
10332 ; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_load:
10333 ; GFX11-WGP: ; %bb.0: ; %entry
10334 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10335 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10336 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10337 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10338 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10339 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10340 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
10341 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10342 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10343 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10344 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10345 ; GFX11-WGP-NEXT: s_endpgm
10346 ;
10347 ; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_load:
10348 ; GFX11-CU: ; %bb.0: ; %entry
10349 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10350 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10351 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10352 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10353 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10354 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10355 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
10356 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10357 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10358 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10359 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10360 ; GFX11-CU-NEXT: s_endpgm
10361 ;
10362 ; GFX12-WGP-LABEL: flat_workgroup_one_as_unordered_load:
10363 ; GFX12-WGP: ; %bb.0: ; %entry
10364 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10365 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10366 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10367 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10368 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10369 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10370 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
10371 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10372 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10373 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10374 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10375 ; GFX12-WGP-NEXT: s_endpgm
10376 ;
10377 ; GFX12-CU-LABEL: flat_workgroup_one_as_unordered_load:
10378 ; GFX12-CU: ; %bb.0: ; %entry
10379 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10380 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10381 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10382 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10383 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10384 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10385 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
10386 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10387 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10388 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10389 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10390 ; GFX12-CU-NEXT: s_endpgm
10391 ptr %in, ptr %out) {
10392 entry:
10393 %val = load atomic i32, ptr %in syncscope("workgroup-one-as") unordered, align 4
10394 store i32 %val, ptr %out
10395 ret void
10396 }
10398 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
10399 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
10400 ; GFX7: ; %bb.0: ; %entry
10401 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
10402 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10403 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
10404 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10405 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10406 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10407 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
10408 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10409 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10410 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10411 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10412 ; GFX7-NEXT: s_endpgm
10413 ;
10414 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
10415 ; GFX10-WGP: ; %bb.0: ; %entry
10416 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
10417 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10418 ; GFX10-WGP-NEXT: s_nop 0
10419 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10420 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10421 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10422 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10423 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
10424 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10425 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10426 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10427 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10428 ; GFX10-WGP-NEXT: s_endpgm
10429 ;
10430 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
10431 ; GFX10-CU: ; %bb.0: ; %entry
10432 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
10433 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10434 ; GFX10-CU-NEXT: s_nop 0
10435 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10436 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10437 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10438 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10439 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
10440 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10441 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10442 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10443 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10444 ; GFX10-CU-NEXT: s_endpgm
10445 ;
10446 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
10447 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10448 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
10449 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10450 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
10451 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10452 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10453 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10454 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
10455 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10456 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10457 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10458 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10459 ; SKIP-CACHE-INV-NEXT: s_endpgm
10460 ;
10461 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
10462 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10463 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10464 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10465 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
10466 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10467 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10468 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10469 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10470 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10471 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10472 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10473 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10474 ;
10475 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
10476 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10477 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10478 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10479 ; GFX90A-TGSPLIT-NEXT: s_nop 0
10480 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10481 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10482 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10483 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
10484 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10485 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10486 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10487 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10488 ;
10489 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
10490 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10491 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10492 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10493 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
10494 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10495 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10496 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10497 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10498 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10499 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10500 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10501 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10502 ;
10503 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_load:
10504 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10505 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10506 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10507 ; GFX940-TGSPLIT-NEXT: s_nop 0
10508 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10509 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10510 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10511 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10512 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10513 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10514 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10515 ; GFX940-TGSPLIT-NEXT: s_endpgm
10516 ;
10517 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
10518 ; GFX11-WGP: ; %bb.0: ; %entry
10519 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10520 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10521 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10522 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10523 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10524 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10525 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
10526 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10527 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10528 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10529 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10530 ; GFX11-WGP-NEXT: s_endpgm
10531 ;
10532 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_load:
10533 ; GFX11-CU: ; %bb.0: ; %entry
10534 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10535 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10536 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10537 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10538 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10539 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10540 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
10541 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10542 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10543 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10544 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10545 ; GFX11-CU-NEXT: s_endpgm
10546 ;
10547 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
10548 ; GFX12-WGP: ; %bb.0: ; %entry
10549 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10550 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10551 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10552 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10553 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10554 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10555 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
10556 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10557 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10558 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10559 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10560 ; GFX12-WGP-NEXT: s_endpgm
10561 ;
10562 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_load:
10563 ; GFX12-CU: ; %bb.0: ; %entry
10564 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10565 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10566 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10567 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10568 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10569 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10570 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
10571 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10572 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10573 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10574 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10575 ; GFX12-CU-NEXT: s_endpgm
10576 ptr %in, ptr %out) {
10577 entry:
10578 %val = load atomic i32, ptr %in syncscope("workgroup-one-as") monotonic, align 4
10579 store i32 %val, ptr %out
10580 ret void
10581 }
10583 define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
10584 ; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
10585 ; GFX7: ; %bb.0: ; %entry
10586 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
10587 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10588 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
10589 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10590 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10591 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10592 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
10593 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10594 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10595 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10596 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10597 ; GFX7-NEXT: s_endpgm
10598 ;
10599 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
10600 ; GFX10-WGP: ; %bb.0: ; %entry
10601 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
10602 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10603 ; GFX10-WGP-NEXT: s_nop 0
10604 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10605 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10606 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10607 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10608 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
10609 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10610 ; GFX10-WGP-NEXT: buffer_gl0_inv
10611 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10612 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10613 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10614 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10615 ; GFX10-WGP-NEXT: s_endpgm
10616 ;
10617 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
10618 ; GFX10-CU: ; %bb.0: ; %entry
10619 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
10620 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10621 ; GFX10-CU-NEXT: s_nop 0
10622 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10623 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10624 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10625 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10626 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
10627 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10628 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10629 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10630 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10631 ; GFX10-CU-NEXT: s_endpgm
10632 ;
10633 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
10634 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10635 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
10636 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10637 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
10638 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10639 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10640 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10641 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
10642 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10643 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10644 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10645 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10646 ; SKIP-CACHE-INV-NEXT: s_endpgm
10647 ;
10648 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
10649 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10650 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10651 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10652 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
10653 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10654 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10655 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10656 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10657 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10658 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10659 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10660 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10661 ;
10662 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
10663 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10664 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10665 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10666 ; GFX90A-TGSPLIT-NEXT: s_nop 0
10667 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10668 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10669 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10670 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
10671 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10672 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10673 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10674 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10675 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10676 ;
10677 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
10678 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10679 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10680 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10681 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
10682 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10683 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10684 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10685 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10686 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10687 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10688 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10689 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10690 ;
10691 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_load:
10692 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10693 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10694 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10695 ; GFX940-TGSPLIT-NEXT: s_nop 0
10696 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10697 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10698 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10699 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10700 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10701 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10702 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10703 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10704 ; GFX940-TGSPLIT-NEXT: s_endpgm
10705 ;
10706 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_load:
10707 ; GFX11-WGP: ; %bb.0: ; %entry
10708 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10709 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10710 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10711 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10712 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10713 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10714 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
10715 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10716 ; GFX11-WGP-NEXT: buffer_gl0_inv
10717 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10718 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10719 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10720 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10721 ; GFX11-WGP-NEXT: s_endpgm
10722 ;
10723 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_load:
10724 ; GFX11-CU: ; %bb.0: ; %entry
10725 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10726 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10727 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10728 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10729 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10730 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10731 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
10732 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10733 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10734 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10735 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10736 ; GFX11-CU-NEXT: s_endpgm
10737 ;
10738 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_load:
10739 ; GFX12-WGP: ; %bb.0: ; %entry
10740 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10741 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10742 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10743 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10744 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10745 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10746 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
10747 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10748 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10749 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10750 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10751 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10752 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10753 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
10754 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10755 ; GFX12-WGP-NEXT: s_endpgm
10756 ;
10757 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_load:
10758 ; GFX12-CU: ; %bb.0: ; %entry
10759 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10760 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10761 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10762 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10763 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10764 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10765 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
10766 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10767 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10768 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10769 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10770 ; GFX12-CU-NEXT: s_endpgm
10771 ptr %in, ptr %out) {
10772 entry:
10773 %val = load atomic i32, ptr %in syncscope("workgroup-one-as") acquire, align 4
10774 store i32 %val, ptr %out
10775 ret void
10776 }
10778 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
10779 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
10780 ; GFX7: ; %bb.0: ; %entry
10781 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
10782 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10783 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
10784 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10785 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10786 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10787 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
10788 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10789 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10790 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10791 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10792 ; GFX7-NEXT: s_endpgm
10793 ;
10794 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
10795 ; GFX10-WGP: ; %bb.0: ; %entry
10796 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
10797 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10798 ; GFX10-WGP-NEXT: s_nop 0
10799 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10800 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10801 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
10802 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10803 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10804 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10805 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
10806 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10807 ; GFX10-WGP-NEXT: buffer_gl0_inv
10808 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10809 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10810 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10811 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10812 ; GFX10-WGP-NEXT: s_endpgm
10813 ;
10814 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
10815 ; GFX10-CU: ; %bb.0: ; %entry
10816 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
10817 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10818 ; GFX10-CU-NEXT: s_nop 0
10819 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10820 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10821 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
10822 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10823 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
10824 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10825 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10826 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10827 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10828 ; GFX10-CU-NEXT: s_endpgm
10829 ;
10830 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
10831 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10832 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
10833 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10834 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
10835 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10836 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
10837 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
10838 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
10839 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10840 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10841 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10842 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10843 ; SKIP-CACHE-INV-NEXT: s_endpgm
10844 ;
10845 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
10846 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10847 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10848 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10849 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
10850 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10851 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10852 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10853 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
10854 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10855 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10856 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10857 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10858 ;
10859 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
10860 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10861 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10862 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10863 ; GFX90A-TGSPLIT-NEXT: s_nop 0
10864 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10865 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10866 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
10867 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10868 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] glc
10869 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10870 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10871 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
10872 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10873 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10874 ;
10875 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
10876 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10877 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10878 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10879 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
10880 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10881 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10882 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10883 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10884 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10885 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10886 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10887 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10888 ;
10889 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_load:
10890 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10891 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10892 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10893 ; GFX940-TGSPLIT-NEXT: s_nop 0
10894 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10895 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10896 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
10897 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10898 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1] sc0
10899 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10900 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10901 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10902 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10903 ; GFX940-TGSPLIT-NEXT: s_endpgm
10904 ;
10905 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
10906 ; GFX11-WGP: ; %bb.0: ; %entry
10907 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10908 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10909 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10910 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10911 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
10912 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10913 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10914 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10915 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
10916 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10917 ; GFX11-WGP-NEXT: buffer_gl0_inv
10918 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
10919 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
10920 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10921 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10922 ; GFX11-WGP-NEXT: s_endpgm
10923 ;
10924 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
10925 ; GFX11-CU: ; %bb.0: ; %entry
10926 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10927 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10928 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10929 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10930 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
10931 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10932 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
10933 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
10934 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
10935 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10936 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10937 ; GFX11-CU-NEXT: s_endpgm
10938 ;
10939 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
10940 ; GFX12-WGP: ; %bb.0: ; %entry
10941 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10942 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10943 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10944 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10945 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
10946 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10947 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10948 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10949 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10950 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10951 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
10952 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10953 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10954 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10955 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10956 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
10957 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
10958 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
10959 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
10960 ; GFX12-WGP-NEXT: s_endpgm
10961 ;
10962 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
10963 ; GFX12-CU: ; %bb.0: ; %entry
10964 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
10965 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10966 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10967 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10968 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
10969 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10970 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
10971 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
10972 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
10973 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
10974 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
10975 ; GFX12-CU-NEXT: s_endpgm
10976 ptr %in, ptr %out) {
10977 entry:
10978 %val = load atomic i32, ptr %in syncscope("workgroup-one-as") seq_cst, align 4
10979 store i32 %val, ptr %out
10980 ret void
10981 }
10983 define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
10984 ; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
10985 ; GFX7: ; %bb.0: ; %entry
10986 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
10987 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
10988 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10989 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10990 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10991 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
10992 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10993 ; GFX7-NEXT: s_endpgm
10994 ;
10995 ; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
10996 ; GFX10-WGP: ; %bb.0: ; %entry
10997 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
10998 ; GFX10-WGP-NEXT: s_nop 0
10999 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11000 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11001 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11002 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11003 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11004 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11005 ; GFX10-WGP-NEXT: s_endpgm
11006 ;
11007 ; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
11008 ; GFX10-CU: ; %bb.0: ; %entry
11009 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
11010 ; GFX10-CU-NEXT: s_nop 0
11011 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11012 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11013 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11014 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11015 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11016 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11017 ; GFX10-CU-NEXT: s_endpgm
11018 ;
11019 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
11020 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11021 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
11022 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
11023 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11024 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11025 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11026 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11027 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11028 ; SKIP-CACHE-INV-NEXT: s_endpgm
11029 ;
11030 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
11031 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11032 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11033 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11034 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11035 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11036 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11037 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11038 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11039 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11040 ;
11041 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
11042 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11043 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11044 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11045 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11046 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11047 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11048 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11049 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11050 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11051 ;
11052 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
11053 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11054 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11055 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11056 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11057 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11058 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11059 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11060 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11061 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11062 ;
11063 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_unordered_store:
11064 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11065 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11066 ; GFX940-TGSPLIT-NEXT: s_nop 0
11067 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11068 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11069 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11070 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11071 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11072 ; GFX940-TGSPLIT-NEXT: s_endpgm
11073 ;
11074 ; GFX11-WGP-LABEL: flat_workgroup_one_as_unordered_store:
11075 ; GFX11-WGP: ; %bb.0: ; %entry
11076 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11077 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11078 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11079 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11080 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11081 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11082 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11083 ; GFX11-WGP-NEXT: s_endpgm
11084 ;
11085 ; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_store:
11086 ; GFX11-CU: ; %bb.0: ; %entry
11087 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11088 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11089 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11090 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11091 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11092 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11093 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11094 ; GFX11-CU-NEXT: s_endpgm
11095 ;
11096 ; GFX12-WGP-LABEL: flat_workgroup_one_as_unordered_store:
11097 ; GFX12-WGP: ; %bb.0: ; %entry
11098 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11099 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11100 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11101 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11102 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11103 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11104 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
11105 ; GFX12-WGP-NEXT: s_endpgm
11106 ;
11107 ; GFX12-CU-LABEL: flat_workgroup_one_as_unordered_store:
11108 ; GFX12-CU: ; %bb.0: ; %entry
11109 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11110 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11111 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11112 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11113 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11114 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11115 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11116 ; GFX12-CU-NEXT: s_endpgm
11117 i32 %in, ptr %out) {
11118 entry:
11119 store atomic i32 %in, ptr %out syncscope("workgroup-one-as") unordered, align 4
11120 ret void
11121 }
11123 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
11124 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
11125 ; GFX7: ; %bb.0: ; %entry
11126 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
11127 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
11128 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11129 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11130 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11131 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11132 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11133 ; GFX7-NEXT: s_endpgm
11134 ;
11135 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
11136 ; GFX10-WGP: ; %bb.0: ; %entry
11137 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
11138 ; GFX10-WGP-NEXT: s_nop 0
11139 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11140 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11141 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11142 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11143 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11144 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11145 ; GFX10-WGP-NEXT: s_endpgm
11146 ;
11147 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
11148 ; GFX10-CU: ; %bb.0: ; %entry
11149 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
11150 ; GFX10-CU-NEXT: s_nop 0
11151 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11152 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11153 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11154 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11155 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11156 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11157 ; GFX10-CU-NEXT: s_endpgm
11158 ;
11159 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
11160 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11161 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
11162 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
11163 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11164 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11165 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11166 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11167 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11168 ; SKIP-CACHE-INV-NEXT: s_endpgm
11169 ;
11170 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
11171 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11172 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11173 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11174 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11175 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11176 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11177 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11178 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11179 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11180 ;
11181 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
11182 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11183 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11184 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11185 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11186 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11187 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11188 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11189 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11190 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11191 ;
11192 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
11193 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11194 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11195 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11196 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11197 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11198 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11199 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11200 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11201 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11202 ;
11203 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_store:
11204 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11205 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11206 ; GFX940-TGSPLIT-NEXT: s_nop 0
11207 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11208 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11209 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11210 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11211 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11212 ; GFX940-TGSPLIT-NEXT: s_endpgm
11213 ;
11214 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
11215 ; GFX11-WGP: ; %bb.0: ; %entry
11216 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11217 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11218 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11219 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11220 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11221 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11222 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11223 ; GFX11-WGP-NEXT: s_endpgm
11224 ;
11225 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_store:
11226 ; GFX11-CU: ; %bb.0: ; %entry
11227 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11228 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11229 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11230 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11231 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11232 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11233 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11234 ; GFX11-CU-NEXT: s_endpgm
11235 ;
11236 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
11237 ; GFX12-WGP: ; %bb.0: ; %entry
11238 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11239 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11240 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11241 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11242 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11243 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11244 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
11245 ; GFX12-WGP-NEXT: s_endpgm
11246 ;
11247 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_store:
11248 ; GFX12-CU: ; %bb.0: ; %entry
11249 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11250 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11251 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11252 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11253 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11254 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11255 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11256 ; GFX12-CU-NEXT: s_endpgm
11257 i32 %in, ptr %out) {
11258 entry:
11259 store atomic i32 %in, ptr %out syncscope("workgroup-one-as") monotonic, align 4
11260 ret void
11261 }
11263 define amdgpu_kernel void @flat_workgroup_one_as_release_store(
11264 ; GFX7-LABEL: flat_workgroup_one_as_release_store:
11265 ; GFX7: ; %bb.0: ; %entry
11266 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
11267 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
11268 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11269 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11270 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11271 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11272 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11273 ; GFX7-NEXT: s_endpgm
11274 ;
11275 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
11276 ; GFX10-WGP: ; %bb.0: ; %entry
11277 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
11278 ; GFX10-WGP-NEXT: s_nop 0
11279 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11280 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11281 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11282 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11283 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11284 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11285 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11286 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11287 ; GFX10-WGP-NEXT: s_endpgm
11288 ;
11289 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
11290 ; GFX10-CU: ; %bb.0: ; %entry
11291 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
11292 ; GFX10-CU-NEXT: s_nop 0
11293 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11294 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11295 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11296 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11297 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11298 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11299 ; GFX10-CU-NEXT: s_endpgm
11300 ;
11301 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
11302 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11303 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
11304 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
11305 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11309 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11310 ; SKIP-CACHE-INV-NEXT: s_endpgm
11311 ;
11312 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
11313 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11314 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11315 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11316 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11317 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11318 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11319 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11320 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11321 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11322 ;
11323 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
11324 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11325 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11326 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11327 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11328 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11329 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11330 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11331 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11332 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11333 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11334 ;
11335 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_store:
11336 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11337 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11338 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11339 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11340 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11341 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11342 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11343 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11344 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11345 ;
11346 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_store:
11347 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11348 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11349 ; GFX940-TGSPLIT-NEXT: s_nop 0
11350 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11351 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11352 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11353 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11354 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11355 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11356 ; GFX940-TGSPLIT-NEXT: s_endpgm
11357 ;
11358 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_store:
11359 ; GFX11-WGP: ; %bb.0: ; %entry
11360 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11361 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11362 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11363 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11364 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11365 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11366 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11367 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11368 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11369 ; GFX11-WGP-NEXT: s_endpgm
11370 ;
11371 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_store:
11372 ; GFX11-CU: ; %bb.0: ; %entry
11373 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11374 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11375 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11376 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11377 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11378 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11379 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11380 ; GFX11-CU-NEXT: s_endpgm
11381 ;
11382 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_store:
11383 ; GFX12-WGP: ; %bb.0: ; %entry
11384 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11385 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11386 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11387 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11388 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11389 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11390 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
11391 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11392 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11393 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11394 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
11395 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
11396 ; GFX12-WGP-NEXT: s_endpgm
11397 ;
11398 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_store:
11399 ; GFX12-CU: ; %bb.0: ; %entry
11400 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11401 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11402 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11403 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11404 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11405 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11406 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11407 ; GFX12-CU-NEXT: s_endpgm
11408 i32 %in, ptr %out) {
11409 entry:
11410 store atomic i32 %in, ptr %out syncscope("workgroup-one-as") release, align 4
11411 ret void
11412 }
11414 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
11415 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
11416 ; GFX7: ; %bb.0: ; %entry
11417 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
11418 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
11419 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11420 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11421 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11422 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11423 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11424 ; GFX7-NEXT: s_endpgm
11425 ;
11426 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
11427 ; GFX10-WGP: ; %bb.0: ; %entry
11428 ; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
11429 ; GFX10-WGP-NEXT: s_nop 0
11430 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11431 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11432 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11433 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11434 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11435 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11436 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11437 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11438 ; GFX10-WGP-NEXT: s_endpgm
11439 ;
11440 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
11441 ; GFX10-CU: ; %bb.0: ; %entry
11442 ; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
11443 ; GFX10-CU-NEXT: s_nop 0
11444 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11445 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11446 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11447 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11448 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11449 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11450 ; GFX10-CU-NEXT: s_endpgm
11451 ;
11452 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
11453 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11454 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
11455 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x2
11456 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11457 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11458 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11459 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11460 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11461 ; SKIP-CACHE-INV-NEXT: s_endpgm
11462 ;
11463 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
11464 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11465 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11466 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11467 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11468 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11469 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11470 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11471 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11472 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11473 ;
11474 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
11475 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11476 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
11477 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11478 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x8
11479 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11480 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11481 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11482 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11483 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11484 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11485 ;
11486 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
11487 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11488 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11489 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11490 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11491 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11492 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11493 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11494 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11495 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11496 ;
11497 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_store:
11498 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11499 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
11500 ; GFX940-TGSPLIT-NEXT: s_nop 0
11501 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x8
11502 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11503 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11504 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11505 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11506 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11507 ; GFX940-TGSPLIT-NEXT: s_endpgm
11508 ;
11509 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
11510 ; GFX11-WGP: ; %bb.0: ; %entry
11511 ; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11512 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11513 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11514 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11515 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11516 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11517 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11518 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11519 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11520 ; GFX11-WGP-NEXT: s_endpgm
11521 ;
11522 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
11523 ; GFX11-CU: ; %bb.0: ; %entry
11524 ; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11525 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11526 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11527 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11528 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11529 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11530 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11531 ; GFX11-CU-NEXT: s_endpgm
11532 ;
11533 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
11534 ; GFX12-WGP: ; %bb.0: ; %entry
11535 ; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
11536 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11537 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11538 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11539 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11540 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11541 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
11542 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11543 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11544 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11545 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
11546 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
11547 ; GFX12-WGP-NEXT: s_endpgm
11548 ;
11549 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
11550 ; GFX12-CU: ; %bb.0: ; %entry
11551 ; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
11552 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
11553 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11554 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11555 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11556 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11557 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
11558 ; GFX12-CU-NEXT: s_endpgm
11559 i32 %in, ptr %out) {
11560 entry:
11561 store atomic i32 %in, ptr %out syncscope("workgroup-one-as") seq_cst, align 4
11562 ret void
11563 }
11565 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
11566 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11567 ; GFX7: ; %bb.0: ; %entry
11568 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11569 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11570 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
11571 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11572 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11573 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11574 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11575 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
11576 ; GFX7-NEXT: s_endpgm
11577 ;
11578 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11579 ; GFX10-WGP: ; %bb.0: ; %entry
11580 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11581 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11582 ; GFX10-WGP-NEXT: s_nop 0
11583 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
11584 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11585 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11586 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11587 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11588 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
11589 ; GFX10-WGP-NEXT: s_endpgm
11590 ;
11591 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11592 ; GFX10-CU: ; %bb.0: ; %entry
11593 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11594 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11595 ; GFX10-CU-NEXT: s_nop 0
11596 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
11597 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11598 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11599 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11600 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11601 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
11602 ; GFX10-CU-NEXT: s_endpgm
11603 ;
11604 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11605 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11606 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
11607 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11608 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
11609 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11610 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11611 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11612 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11613 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
11614 ; SKIP-CACHE-INV-NEXT: s_endpgm
11615 ;
11616 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11617 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11618 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11619 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11620 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11621 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11622 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11623 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11624 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11625 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11626 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11627 ;
11628 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11629 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11630 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11631 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11632 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11633 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11634 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11635 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11636 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11637 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11638 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11639 ;
11640 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11641 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11642 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11643 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11644 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11645 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11646 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11647 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11648 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11649 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11650 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11651 ;
11652 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11653 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11654 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11655 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11656 ; GFX940-TGSPLIT-NEXT: s_nop 0
11657 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11658 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11659 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11660 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11661 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11662 ; GFX940-TGSPLIT-NEXT: s_endpgm
11663 ;
11664 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11665 ; GFX11-WGP: ; %bb.0: ; %entry
11666 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11667 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11668 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
11669 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11670 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11671 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11672 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11673 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
11674 ; GFX11-WGP-NEXT: s_endpgm
11675 ;
11676 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11677 ; GFX11-CU: ; %bb.0: ; %entry
11678 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11679 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11680 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
11681 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11682 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11683 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11684 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11685 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
11686 ; GFX11-CU-NEXT: s_endpgm
11687 ;
11688 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11689 ; GFX12-WGP: ; %bb.0: ; %entry
11690 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11691 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11692 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
11693 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11694 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11695 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11696 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11697 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
11698 ; GFX12-WGP-NEXT: s_endpgm
11699 ;
11700 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
11701 ; GFX12-CU: ; %bb.0: ; %entry
11702 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11703 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11704 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
11705 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11706 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11707 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11708 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11709 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
11710 ; GFX12-CU-NEXT: s_endpgm
11711 ptr %out, i32 %in) {
11712 entry:
11713 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") monotonic
11714 ret void
11715 }
11717 define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
11718 ; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11719 ; GFX7: ; %bb.0: ; %entry
11720 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11721 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11722 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
11723 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11724 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11725 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11726 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11727 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
11728 ; GFX7-NEXT: s_endpgm
11729 ;
11730 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11731 ; GFX10-WGP: ; %bb.0: ; %entry
11732 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11733 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11734 ; GFX10-WGP-NEXT: s_nop 0
11735 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
11736 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11737 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11738 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11739 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11740 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
11741 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11742 ; GFX10-WGP-NEXT: buffer_gl0_inv
11743 ; GFX10-WGP-NEXT: s_endpgm
11744 ;
11745 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11746 ; GFX10-CU: ; %bb.0: ; %entry
11747 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11748 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11749 ; GFX10-CU-NEXT: s_nop 0
11750 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
11751 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11752 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11753 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11754 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11755 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
11756 ; GFX10-CU-NEXT: s_endpgm
11757 ;
11758 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11759 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11760 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
11761 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11762 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
11763 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11764 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11765 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11766 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11767 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
11768 ; SKIP-CACHE-INV-NEXT: s_endpgm
11769 ;
11770 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11771 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11772 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11773 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11774 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11775 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11776 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11777 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11778 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11779 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11780 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11781 ;
11782 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11783 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11784 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11785 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11786 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11787 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11788 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11789 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11790 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11791 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11792 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11793 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11794 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11795 ;
11796 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11797 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11798 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11799 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11800 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11801 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11802 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11803 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11804 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11805 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11806 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11807 ;
11808 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11809 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11810 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11811 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11812 ; GFX940-TGSPLIT-NEXT: s_nop 0
11813 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11814 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11815 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11816 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11817 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11818 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11819 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
11820 ; GFX940-TGSPLIT-NEXT: s_endpgm
11821 ;
11822 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11823 ; GFX11-WGP: ; %bb.0: ; %entry
11824 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11825 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11826 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
11827 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11828 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11829 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11830 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11831 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
11832 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11833 ; GFX11-WGP-NEXT: buffer_gl0_inv
11834 ; GFX11-WGP-NEXT: s_endpgm
11835 ;
11836 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11837 ; GFX11-CU: ; %bb.0: ; %entry
11838 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11839 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11840 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
11841 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11842 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
11843 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
11844 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
11845 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
11846 ; GFX11-CU-NEXT: s_endpgm
11847 ;
11848 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11849 ; GFX12-WGP: ; %bb.0: ; %entry
11850 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11851 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11852 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
11853 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11854 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
11855 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
11856 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
11857 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
11858 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
11859 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
11860 ; GFX12-WGP-NEXT: s_endpgm
11861 ;
11862 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
11863 ; GFX12-CU: ; %bb.0: ; %entry
11864 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11865 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11866 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
11867 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11868 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
11869 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
11870 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
11871 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
11872 ; GFX12-CU-NEXT: s_endpgm
11873 ptr %out, i32 %in) {
11874 entry:
11875 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
11876 ret void
11877 }
11879 define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
11880 ; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
11881 ; GFX7: ; %bb.0: ; %entry
11882 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11883 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11884 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
11885 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11886 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11887 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11888 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11889 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
11890 ; GFX7-NEXT: s_endpgm
11891 ;
11892 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
11893 ; GFX10-WGP: ; %bb.0: ; %entry
11894 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11895 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11896 ; GFX10-WGP-NEXT: s_nop 0
11897 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
11898 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11899 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
11900 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
11901 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
11902 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11903 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11904 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
11905 ; GFX10-WGP-NEXT: s_endpgm
11906 ;
11907 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
11908 ; GFX10-CU: ; %bb.0: ; %entry
11909 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11910 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11911 ; GFX10-CU-NEXT: s_nop 0
11912 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
11913 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11914 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
11915 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
11916 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
11917 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
11918 ; GFX10-CU-NEXT: s_endpgm
11919 ;
11920 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
11921 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11922 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
11923 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11924 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
11925 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11926 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
11927 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
11928 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
11929 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
11930 ; SKIP-CACHE-INV-NEXT: s_endpgm
11931 ;
11932 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
11933 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11934 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11935 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11936 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11937 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11938 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11939 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11940 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11941 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11942 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11943 ;
11944 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
11945 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11946 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11947 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11948 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11949 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
11950 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11951 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
11952 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
11953 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11954 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11955 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11956 ;
11957 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
11958 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11959 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11960 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11961 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11962 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11963 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11964 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11965 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11966 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11967 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11968 ;
11969 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_atomicrmw:
11970 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11971 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11972 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11973 ; GFX940-TGSPLIT-NEXT: s_nop 0
11974 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
11975 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11976 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
11977 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
11978 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11979 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
11980 ; GFX940-TGSPLIT-NEXT: s_endpgm
11981 ;
11982 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
11983 ; GFX11-WGP: ; %bb.0: ; %entry
11984 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11985 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11986 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
11987 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11988 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
11989 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
11990 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
11991 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11992 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11993 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
11994 ; GFX11-WGP-NEXT: s_endpgm
11995 ;
11996 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
11997 ; GFX11-CU: ; %bb.0: ; %entry
11998 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11999 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12000 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12001 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12002 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12003 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12004 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12005 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12006 ; GFX11-CU-NEXT: s_endpgm
12007 ;
12008 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
12009 ; GFX12-WGP: ; %bb.0: ; %entry
12010 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12011 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12012 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
12013 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12014 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12015 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12016 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12017 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12018 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12019 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12020 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12021 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12022 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
12023 ; GFX12-WGP-NEXT: s_endpgm
12024 ;
12025 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
12026 ; GFX12-CU: ; %bb.0: ; %entry
12027 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12028 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12029 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12030 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12031 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12032 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12033 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12034 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12035 ; GFX12-CU-NEXT: s_endpgm
12036 ptr %out, i32 %in) {
12037 entry:
12038 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") release
12039 ret void
12040 }
12042 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
12043 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12044 ; GFX7: ; %bb.0: ; %entry
12045 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12046 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12047 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12048 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12049 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12050 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12051 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12052 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12053 ; GFX7-NEXT: s_endpgm
12054 ;
12055 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12056 ; GFX10-WGP: ; %bb.0: ; %entry
12057 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
12058 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12059 ; GFX10-WGP-NEXT: s_nop 0
12060 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
12061 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12062 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12063 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12064 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12065 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12066 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12067 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
12068 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12069 ; GFX10-WGP-NEXT: buffer_gl0_inv
12070 ; GFX10-WGP-NEXT: s_endpgm
12071 ;
12072 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12073 ; GFX10-CU: ; %bb.0: ; %entry
12074 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
12075 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12076 ; GFX10-CU-NEXT: s_nop 0
12077 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
12078 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12079 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12080 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12081 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12082 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
12083 ; GFX10-CU-NEXT: s_endpgm
12084 ;
12085 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12086 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12087 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
12088 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12089 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
12090 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12091 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12092 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12093 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12094 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
12095 ; SKIP-CACHE-INV-NEXT: s_endpgm
12096 ;
12097 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12098 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12099 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12100 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12101 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12102 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
12103 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12104 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12105 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12106 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12107 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12108 ;
12109 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12110 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12111 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12112 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12113 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12114 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
12115 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12116 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12117 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12118 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12119 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12120 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12121 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12122 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12123 ;
12124 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12125 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12126 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12127 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12128 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12129 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
12130 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12131 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12132 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12133 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12134 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12135 ;
12136 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12137 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12138 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12139 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12140 ; GFX940-TGSPLIT-NEXT: s_nop 0
12141 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
12142 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12143 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12144 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12145 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12146 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12147 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12148 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12149 ; GFX940-TGSPLIT-NEXT: s_endpgm
12150 ;
12151 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12152 ; GFX11-WGP: ; %bb.0: ; %entry
12153 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12154 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12155 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
12156 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12157 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12158 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12159 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12160 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12161 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12162 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
12163 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12164 ; GFX11-WGP-NEXT: buffer_gl0_inv
12165 ; GFX11-WGP-NEXT: s_endpgm
12166 ;
12167 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12168 ; GFX11-CU: ; %bb.0: ; %entry
12169 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12170 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12171 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12172 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12173 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12174 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12175 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12176 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12177 ; GFX11-CU-NEXT: s_endpgm
12178 ;
12179 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12180 ; GFX12-WGP: ; %bb.0: ; %entry
12181 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12182 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12183 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
12184 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12185 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12186 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12187 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12188 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12189 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12190 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12191 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12192 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12193 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
12194 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12195 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12196 ; GFX12-WGP-NEXT: s_endpgm
12197 ;
12198 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
12199 ; GFX12-CU: ; %bb.0: ; %entry
12200 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12201 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12202 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12203 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12204 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12205 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12206 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12207 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12208 ; GFX12-CU-NEXT: s_endpgm
12209 ptr %out, i32 %in) {
12210 entry:
12211 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
12212 ret void
12213 }
12215 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
12216 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12217 ; GFX7: ; %bb.0: ; %entry
12218 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12219 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12220 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12221 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12222 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12223 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12224 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12225 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12226 ; GFX7-NEXT: s_endpgm
12227 ;
12228 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12229 ; GFX10-WGP: ; %bb.0: ; %entry
12230 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
12231 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12232 ; GFX10-WGP-NEXT: s_nop 0
12233 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
12234 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12235 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
12236 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
12237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
12238 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12239 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12240 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
12241 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12242 ; GFX10-WGP-NEXT: buffer_gl0_inv
12243 ; GFX10-WGP-NEXT: s_endpgm
12244 ;
12245 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12246 ; GFX10-CU: ; %bb.0: ; %entry
12247 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
12248 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12249 ; GFX10-CU-NEXT: s_nop 0
12250 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
12251 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12252 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
12253 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
12254 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
12255 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
12256 ; GFX10-CU-NEXT: s_endpgm
12257 ;
12258 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12259 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12260 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
12261 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12262 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
12263 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12264 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
12265 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
12266 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
12267 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
12268 ; SKIP-CACHE-INV-NEXT: s_endpgm
12269 ;
12270 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12271 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12272 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12273 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12274 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12275 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
12276 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12277 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12278 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12279 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12280 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12281 ;
12282 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12283 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12284 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12285 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12286 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12287 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
12288 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12289 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]
12290 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
12291 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12292 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12293 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12294 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12295 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12296 ;
12297 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12298 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12299 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12300 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12301 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12302 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
12303 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12304 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12305 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12306 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12307 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12308 ;
12309 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12310 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12311 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12312 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
12313 ; GFX940-TGSPLIT-NEXT: s_nop 0
12314 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
12315 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12316 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
12317 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s0
12318 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12319 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
12320 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12321 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12322 ; GFX940-TGSPLIT-NEXT: s_endpgm
12323 ;
12324 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12325 ; GFX11-WGP: ; %bb.0: ; %entry
12326 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12327 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12328 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
12329 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12330 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
12331 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
12332 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
12333 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12334 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12335 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
12336 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12337 ; GFX11-WGP-NEXT: buffer_gl0_inv
12338 ; GFX11-WGP-NEXT: s_endpgm
12339 ;
12340 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12341 ; GFX11-CU: ; %bb.0: ; %entry
12342 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12343 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12344 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12345 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12346 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
12347 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
12348 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
12349 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12350 ; GFX11-CU-NEXT: s_endpgm
12351 ;
12352 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12353 ; GFX12-WGP: ; %bb.0: ; %entry
12354 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12355 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12356 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
12357 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12358 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
12359 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
12360 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
12361 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12362 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12363 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12364 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12365 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12366 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SE
12367 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12368 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12369 ; GFX12-WGP-NEXT: s_endpgm
12370 ;
12371 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
12372 ; GFX12-CU: ; %bb.0: ; %entry
12373 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12374 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
12375 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
12376 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12377 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
12378 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
12379 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
12380 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
12381 ; GFX12-CU-NEXT: s_endpgm
12382 ptr %out, i32 %in) {
12383 entry:
12384 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
12385 ret void
12386 }
12388 define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
12389 ; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12390 ; GFX7: ; %bb.0: ; %entry
12391 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12392 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
12393 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12394 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12395 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12396 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
12397 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12398 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12399 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12400 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12401 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12402 ; GFX7-NEXT: s_endpgm
12403 ;
12404 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12405 ; GFX10-WGP: ; %bb.0: ; %entry
12406 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12407 ; GFX10-WGP-NEXT: s_nop 0
12408 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12409 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12410 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12411 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12412 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
12413 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12414 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12415 ; GFX10-WGP-NEXT: buffer_gl0_inv
12416 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12417 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12418 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12419 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12420 ; GFX10-WGP-NEXT: s_endpgm
12421 ;
12422 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12423 ; GFX10-CU: ; %bb.0: ; %entry
12424 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12425 ; GFX10-CU-NEXT: s_nop 0
12426 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12427 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12428 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12429 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12430 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
12431 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12432 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12433 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12434 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12435 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12436 ; GFX10-CU-NEXT: s_endpgm
12437 ;
12438 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12439 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12440 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12441 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
12442 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12443 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12444 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12445 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
12446 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12447 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12448 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12449 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12450 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12451 ; SKIP-CACHE-INV-NEXT: s_endpgm
12452 ;
12453 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12454 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12455 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12456 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12457 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12458 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12459 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12460 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12461 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12462 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12463 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12464 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12465 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12466 ;
12467 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12468 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12469 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12470 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12471 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12472 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12473 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12474 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12475 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12476 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12477 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12478 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12479 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12480 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12481 ;
12482 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12483 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12484 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12485 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12486 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12487 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12488 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12489 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12490 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12491 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12492 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12493 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12494 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12495 ;
12496 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12497 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12498 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12499 ; GFX940-TGSPLIT-NEXT: s_nop 0
12500 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12501 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12502 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12503 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12504 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12505 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12506 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12507 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12508 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12509 ; GFX940-TGSPLIT-NEXT: s_endpgm
12510 ;
12511 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12512 ; GFX11-WGP: ; %bb.0: ; %entry
12513 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12514 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12515 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12516 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12517 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12518 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
12519 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12520 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12521 ; GFX11-WGP-NEXT: buffer_gl0_inv
12522 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12523 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12524 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12525 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12526 ; GFX11-WGP-NEXT: s_endpgm
12527 ;
12528 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12529 ; GFX11-CU: ; %bb.0: ; %entry
12530 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12531 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12532 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12533 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12534 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12535 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
12536 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12537 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12538 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12539 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12540 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12541 ; GFX11-CU-NEXT: s_endpgm
12542 ;
12543 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12544 ; GFX12-WGP: ; %bb.0: ; %entry
12545 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12546 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12547 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12548 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12549 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12550 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
12551 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
12552 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12553 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12554 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12555 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12556 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12557 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12558 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
12559 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
12560 ; GFX12-WGP-NEXT: s_endpgm
12561 ;
12562 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
12563 ; GFX12-CU: ; %bb.0: ; %entry
12564 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12565 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12566 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12567 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12568 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12569 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
12570 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
12571 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12572 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12573 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
12574 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
12575 ; GFX12-CU-NEXT: s_endpgm
12576 ptr %out, i32 %in) {
12577 entry:
12578 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
12579 store i32 %val, ptr %out, align 4
12580 ret void
12581 }
12583 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
12584 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12585 ; GFX7: ; %bb.0: ; %entry
12586 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12587 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
12588 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12589 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12590 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12591 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
12592 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12593 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12594 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12595 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12596 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12597 ; GFX7-NEXT: s_endpgm
12598 ;
12599 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12600 ; GFX10-WGP: ; %bb.0: ; %entry
12601 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12602 ; GFX10-WGP-NEXT: s_nop 0
12603 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12604 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12605 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12606 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12607 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
12608 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12609 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12610 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12611 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12612 ; GFX10-WGP-NEXT: buffer_gl0_inv
12613 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12614 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12615 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12616 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12617 ; GFX10-WGP-NEXT: s_endpgm
12618 ;
12619 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12620 ; GFX10-CU: ; %bb.0: ; %entry
12621 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12622 ; GFX10-CU-NEXT: s_nop 0
12623 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12624 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12625 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12626 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12627 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
12628 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12629 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12630 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12631 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12632 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12633 ; GFX10-CU-NEXT: s_endpgm
12634 ;
12635 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12636 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12637 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12638 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
12639 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12640 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12641 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12642 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
12643 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12644 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12645 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12646 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12647 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12648 ; SKIP-CACHE-INV-NEXT: s_endpgm
12649 ;
12650 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12651 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12652 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12653 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12654 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12655 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12656 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12657 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12658 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12659 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12660 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12661 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12662 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12663 ;
12664 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12665 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12666 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12667 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12668 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12669 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12670 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12671 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12672 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12673 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12674 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12675 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12676 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12677 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12678 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12679 ;
12680 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12681 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12682 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12683 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12684 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12685 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12686 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12687 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12688 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12689 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12690 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12691 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12692 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12693 ;
12694 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12695 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12696 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12697 ; GFX940-TGSPLIT-NEXT: s_nop 0
12698 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12699 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12700 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12701 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12702 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12703 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12704 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12705 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12706 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12707 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12708 ; GFX940-TGSPLIT-NEXT: s_endpgm
12709 ;
12710 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12711 ; GFX11-WGP: ; %bb.0: ; %entry
12712 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12713 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12714 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12715 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12716 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12717 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
12718 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12719 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12720 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12721 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12722 ; GFX11-WGP-NEXT: buffer_gl0_inv
12723 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12724 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12725 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12726 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12727 ; GFX11-WGP-NEXT: s_endpgm
12728 ;
12729 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12730 ; GFX11-CU: ; %bb.0: ; %entry
12731 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12732 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12733 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12734 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12735 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12736 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
12737 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12738 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12739 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12740 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12741 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12742 ; GFX11-CU-NEXT: s_endpgm
12743 ;
12744 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12745 ; GFX12-WGP: ; %bb.0: ; %entry
12746 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12747 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12748 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12749 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12750 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12751 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
12752 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12753 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12754 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12755 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12756 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12757 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
12758 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12759 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12760 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12761 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12762 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12763 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12764 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
12765 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
12766 ; GFX12-WGP-NEXT: s_endpgm
12767 ;
12768 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
12769 ; GFX12-CU: ; %bb.0: ; %entry
12770 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12771 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12772 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12773 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12774 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12775 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
12776 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
12777 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12778 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12779 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
12780 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
12781 ; GFX12-CU-NEXT: s_endpgm
12782 ptr %out, i32 %in) {
12783 entry:
12784 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
12785 store i32 %val, ptr %out, align 4
12786 ret void
12787 }
12789 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
12790 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12791 ; GFX7: ; %bb.0: ; %entry
12792 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12793 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
12794 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12795 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12796 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12797 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
12798 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12799 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
12800 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
12801 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12802 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12803 ; GFX7-NEXT: s_endpgm
12804 ;
12805 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12806 ; GFX10-WGP: ; %bb.0: ; %entry
12807 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12808 ; GFX10-WGP-NEXT: s_nop 0
12809 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12810 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12811 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12812 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12813 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s6
12814 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12815 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12816 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12817 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12818 ; GFX10-WGP-NEXT: buffer_gl0_inv
12819 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
12820 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
12821 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12822 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
12823 ; GFX10-WGP-NEXT: s_endpgm
12824 ;
12825 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12826 ; GFX10-CU: ; %bb.0: ; %entry
12827 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12828 ; GFX10-CU-NEXT: s_nop 0
12829 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12830 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12831 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12832 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12833 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s6
12834 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12835 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
12836 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
12837 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12838 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
12839 ; GFX10-CU-NEXT: s_endpgm
12840 ;
12841 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12842 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12843 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12844 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[2:3], 0x2
12845 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12846 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12847 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12848 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
12849 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12850 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
12851 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
12852 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12853 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
12854 ; SKIP-CACHE-INV-NEXT: s_endpgm
12855 ;
12856 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12857 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12858 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12859 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12860 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12861 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12862 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12863 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12864 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12865 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12866 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12867 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
12868 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12869 ;
12870 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12871 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12872 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12873 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12874 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12875 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12876 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12877 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s6
12878 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12879 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
12880 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12881 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12882 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
12883 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
12884 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12885 ;
12886 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12887 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12888 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12889 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12890 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12891 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12892 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12893 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12894 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12895 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12896 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12897 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12898 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12899 ;
12900 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12901 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12902 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12903 ; GFX940-TGSPLIT-NEXT: s_nop 0
12904 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12905 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12906 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12907 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
12908 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12909 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
12910 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12911 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12912 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
12913 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
12914 ; GFX940-TGSPLIT-NEXT: s_endpgm
12915 ;
12916 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12917 ; GFX11-WGP: ; %bb.0: ; %entry
12918 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12919 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12920 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12921 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12922 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12923 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s2
12924 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12925 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12926 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12927 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12928 ; GFX11-WGP-NEXT: buffer_gl0_inv
12929 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
12930 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
12931 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12932 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
12933 ; GFX11-WGP-NEXT: s_endpgm
12934 ;
12935 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12936 ; GFX11-CU: ; %bb.0: ; %entry
12937 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12938 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12939 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12940 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12941 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12942 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s2
12943 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
12944 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
12945 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
12946 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
12947 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
12948 ; GFX11-CU-NEXT: s_endpgm
12949 ;
12950 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12951 ; GFX12-WGP: ; %bb.0: ; %entry
12952 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12953 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12954 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12955 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12956 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12957 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s2
12958 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12959 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12960 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12961 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12962 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12963 ; GFX12-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE
12964 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12965 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12966 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12967 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12968 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
12969 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
12970 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
12971 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
12972 ; GFX12-WGP-NEXT: s_endpgm
12973 ;
12974 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
12975 ; GFX12-CU: ; %bb.0: ; %entry
12976 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12977 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12978 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12979 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12980 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12981 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s2
12982 ; GFX12-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 th:TH_ATOMIC_RETURN
12983 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
12984 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
12985 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
12986 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
12987 ; GFX12-CU-NEXT: s_endpgm
12988 ptr %out, i32 %in) {
12989 entry:
12990 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
12991 store i32 %val, ptr %out, align 4
12992 ret void
12993 }
12995 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
12996 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
12997 ; GFX7: ; %bb.0: ; %entry
12998 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12999 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13000 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13001 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13002 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13003 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13004 ; GFX7-NEXT: s_mov_b32 s4, s8
13005 ; GFX7-NEXT: s_mov_b32 s5, s9
13006 ; GFX7-NEXT: s_mov_b32 s9, s10
13007 ; GFX7-NEXT: s_mov_b32 s8, s11
13008 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13009 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13010 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13011 ; GFX7-NEXT: s_mov_b32 s5, s8
13012 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13013 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13014 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13015 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13016 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13017 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13018 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13019 ; GFX7-NEXT: s_endpgm
13020 ;
13021 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13022 ; GFX10-WGP: ; %bb.0: ; %entry
13023 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
13024 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13025 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
13026 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
13027 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
13028 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13029 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
13030 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
13031 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
13032 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
13033 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
13034 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
13035 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13036 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
13037 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
13038 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13039 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13040 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
13041 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13042 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13043 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13044 ; GFX10-WGP-NEXT: s_endpgm
13045 ;
13046 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13047 ; GFX10-CU: ; %bb.0: ; %entry
13048 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
13049 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13050 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
13051 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
13052 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
13053 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13054 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
13055 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
13056 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
13057 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
13058 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
13059 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
13060 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13061 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
13062 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
13063 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13064 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13065 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
13066 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13067 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13068 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13069 ; GFX10-CU-NEXT: s_endpgm
13070 ;
13071 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13072 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13073 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
13074 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
13075 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
13076 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
13077 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
13078 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13079 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
13080 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
13081 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
13082 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
13083 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
13084 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
13085 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
13086 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
13087 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
13088 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13089 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13090 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
13091 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13092 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13093 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13094 ; SKIP-CACHE-INV-NEXT: s_endpgm
13095 ;
13096 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13097 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13098 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13099 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13100 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13101 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13102 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13103 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13104 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13105 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13106 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13107 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13108 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13109 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13110 ;
13111 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13112 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13113 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13114 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13115 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13116 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13117 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13118 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13119 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13120 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13121 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13122 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13123 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13124 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13125 ;
13126 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13127 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13128 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13129 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13130 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13131 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13132 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13133 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13134 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13135 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13136 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13137 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13138 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13139 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13140 ;
13141 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13142 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13143 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13144 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13145 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13146 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13147 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13148 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13149 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13150 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13151 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13152 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13153 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13154 ; GFX940-TGSPLIT-NEXT: s_endpgm
13155 ;
13156 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13157 ; GFX11-WGP: ; %bb.0: ; %entry
13158 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13159 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13160 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13161 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13162 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13163 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
13164 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13165 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13166 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
13167 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13168 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13169 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13170 ; GFX11-WGP-NEXT: s_endpgm
13171 ;
13172 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13173 ; GFX11-CU: ; %bb.0: ; %entry
13174 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13175 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13176 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13177 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13178 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13179 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
13180 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13181 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13182 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
13183 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13184 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13185 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13186 ; GFX11-CU-NEXT: s_endpgm
13187 ;
13188 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13189 ; GFX12-WGP: ; %bb.0: ; %entry
13190 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13191 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13192 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13193 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13194 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13195 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
13196 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13197 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13198 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
13199 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13200 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13201 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
13202 ; GFX12-WGP-NEXT: s_endpgm
13203 ;
13204 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
13205 ; GFX12-CU: ; %bb.0: ; %entry
13206 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13207 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13208 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13209 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13210 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13211 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
13212 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13213 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13214 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
13215 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13216 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13217 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13218 ; GFX12-CU-NEXT: s_endpgm
13219 ptr %out, i32 %in, i32 %old) {
13220 entry:
13221 %gep = getelementptr i32, ptr %out, i32 4
13222 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
13223 ret void
13224 }
13226 define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
13227 ; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13228 ; GFX7: ; %bb.0: ; %entry
13229 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13230 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13231 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13232 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13233 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13234 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13235 ; GFX7-NEXT: s_mov_b32 s4, s8
13236 ; GFX7-NEXT: s_mov_b32 s5, s9
13237 ; GFX7-NEXT: s_mov_b32 s9, s10
13238 ; GFX7-NEXT: s_mov_b32 s8, s11
13239 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13240 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13241 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13242 ; GFX7-NEXT: s_mov_b32 s5, s8
13243 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13244 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13245 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13246 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13247 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13248 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13249 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13250 ; GFX7-NEXT: s_endpgm
13251 ;
13252 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13253 ; GFX10-WGP: ; %bb.0: ; %entry
13254 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
13255 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13256 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
13257 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
13258 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
13259 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13260 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
13261 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
13262 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
13263 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
13264 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
13265 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
13266 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13267 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
13268 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
13269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13270 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13271 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
13272 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13273 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13274 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13275 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13276 ; GFX10-WGP-NEXT: buffer_gl0_inv
13277 ; GFX10-WGP-NEXT: s_endpgm
13278 ;
13279 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13280 ; GFX10-CU: ; %bb.0: ; %entry
13281 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
13282 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13283 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
13284 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
13285 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
13286 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13287 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
13288 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
13289 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
13290 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
13291 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
13292 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
13293 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13294 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
13295 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
13296 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13297 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13298 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
13299 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13300 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13301 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13302 ; GFX10-CU-NEXT: s_endpgm
13303 ;
13304 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13305 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13306 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
13307 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
13308 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
13309 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
13310 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
13311 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13312 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
13313 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
13314 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
13315 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
13316 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
13317 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
13318 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
13319 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
13320 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
13321 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13322 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13323 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
13324 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13325 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13326 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13327 ; SKIP-CACHE-INV-NEXT: s_endpgm
13328 ;
13329 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13330 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13331 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13332 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13333 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13334 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13335 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13336 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13337 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13338 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13339 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13340 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13341 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13342 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13343 ;
13344 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13345 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13346 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13347 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13348 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13349 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13350 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13351 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13352 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13353 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13354 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13355 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13356 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13357 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13358 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13359 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13360 ;
13361 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13362 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13363 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13364 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13365 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13366 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13368 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13369 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13370 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13371 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13372 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13373 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13374 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13375 ;
13376 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13377 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13378 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13379 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13380 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13381 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13382 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13383 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13384 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13385 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13386 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13387 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13388 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13389 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13390 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13391 ; GFX940-TGSPLIT-NEXT: s_endpgm
13392 ;
13393 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13394 ; GFX11-WGP: ; %bb.0: ; %entry
13395 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13396 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13397 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13398 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13399 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
13401 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13402 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13403 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
13404 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13405 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13406 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13407 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13408 ; GFX11-WGP-NEXT: buffer_gl0_inv
13409 ; GFX11-WGP-NEXT: s_endpgm
13410 ;
13411 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13412 ; GFX11-CU: ; %bb.0: ; %entry
13413 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13414 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13415 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13416 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13417 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13418 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
13419 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13420 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13421 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
13422 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13423 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13424 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13425 ; GFX11-CU-NEXT: s_endpgm
13426 ;
13427 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13428 ; GFX12-WGP: ; %bb.0: ; %entry
13429 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13430 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13431 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13432 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13433 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13434 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
13435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13436 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13437 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
13438 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13439 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13440 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
13441 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13442 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13443 ; GFX12-WGP-NEXT: s_endpgm
13444 ;
13445 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
13446 ; GFX12-CU: ; %bb.0: ; %entry
13447 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13448 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13449 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13450 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13451 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13452 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
13453 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13454 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13455 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
13456 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13457 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13458 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13459 ; GFX12-CU-NEXT: s_endpgm
13460 ptr %out, i32 %in, i32 %old) {
13461 entry:
13462 %gep = getelementptr i32, ptr %out, i32 4
13463 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
13464 ret void
13465 }
13467 define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
13468 ; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13469 ; GFX7: ; %bb.0: ; %entry
13470 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13471 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13472 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13473 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13474 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13475 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13476 ; GFX7-NEXT: s_mov_b32 s4, s8
13477 ; GFX7-NEXT: s_mov_b32 s5, s9
13478 ; GFX7-NEXT: s_mov_b32 s9, s10
13479 ; GFX7-NEXT: s_mov_b32 s8, s11
13480 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13481 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13482 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13483 ; GFX7-NEXT: s_mov_b32 s5, s8
13484 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13485 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13486 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13487 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13488 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13489 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13490 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13491 ; GFX7-NEXT: s_endpgm
13492 ;
13493 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13494 ; GFX10-WGP: ; %bb.0: ; %entry
13495 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
13496 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13497 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
13498 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
13499 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
13500 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13501 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
13502 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
13503 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
13504 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
13505 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
13506 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
13507 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13508 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
13509 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
13510 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13511 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13512 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
13513 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13514 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13515 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13516 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13517 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13518 ; GFX10-WGP-NEXT: s_endpgm
13519 ;
13520 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13521 ; GFX10-CU: ; %bb.0: ; %entry
13522 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
13523 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13524 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
13525 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
13526 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
13527 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13528 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
13529 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
13530 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
13531 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
13532 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
13533 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
13534 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13535 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
13536 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
13537 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13538 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13539 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
13540 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13541 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13542 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13543 ; GFX10-CU-NEXT: s_endpgm
13544 ;
13545 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13546 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13547 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
13548 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
13549 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
13550 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
13551 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
13552 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13553 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
13554 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
13555 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
13556 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
13557 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
13558 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
13559 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
13560 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
13561 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
13562 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13563 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13564 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
13565 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13566 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13567 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13568 ; SKIP-CACHE-INV-NEXT: s_endpgm
13569 ;
13570 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13571 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13572 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13573 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13574 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13575 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13576 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13577 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13578 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13579 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13580 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13581 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13582 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13583 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13584 ;
13585 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13586 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13587 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13588 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13589 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13590 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13591 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13592 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13593 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13594 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13595 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13596 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13597 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13598 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13599 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13600 ;
13601 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13602 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13603 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13604 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13605 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13606 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13607 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13608 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13609 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13610 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13611 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13612 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13613 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13614 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13615 ;
13616 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13617 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13618 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13619 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13620 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13621 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13622 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13623 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13624 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13625 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13626 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13627 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13628 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13629 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13630 ; GFX940-TGSPLIT-NEXT: s_endpgm
13631 ;
13632 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13633 ; GFX11-WGP: ; %bb.0: ; %entry
13634 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13635 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13636 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13637 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13638 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13639 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
13640 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13641 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13642 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
13643 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13644 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13645 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13646 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13647 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13648 ; GFX11-WGP-NEXT: s_endpgm
13649 ;
13650 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13651 ; GFX11-CU: ; %bb.0: ; %entry
13652 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13653 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13654 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13655 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13656 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13657 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
13658 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13659 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13660 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
13661 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13662 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13663 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13664 ; GFX11-CU-NEXT: s_endpgm
13665 ;
13666 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13667 ; GFX12-WGP: ; %bb.0: ; %entry
13668 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13669 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13670 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13671 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13672 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13673 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
13674 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13675 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13676 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
13677 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13678 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13679 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
13680 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13681 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13682 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13683 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13684 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
13685 ; GFX12-WGP-NEXT: s_endpgm
13686 ;
13687 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
13688 ; GFX12-CU: ; %bb.0: ; %entry
13689 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13690 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13691 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13692 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13693 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13694 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
13695 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13696 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13697 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
13698 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13699 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13700 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13701 ; GFX12-CU-NEXT: s_endpgm
13702 ptr %out, i32 %in, i32 %old) {
13703 entry:
13704 %gep = getelementptr i32, ptr %out, i32 4
13705 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
13706 ret void
13707 }
13709 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
13710 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13711 ; GFX7: ; %bb.0: ; %entry
13712 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13713 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13714 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13715 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13716 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13717 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13718 ; GFX7-NEXT: s_mov_b32 s4, s8
13719 ; GFX7-NEXT: s_mov_b32 s5, s9
13720 ; GFX7-NEXT: s_mov_b32 s9, s10
13721 ; GFX7-NEXT: s_mov_b32 s8, s11
13722 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13723 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13724 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13725 ; GFX7-NEXT: s_mov_b32 s5, s8
13726 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13727 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13728 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13729 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13730 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13731 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13732 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13733 ; GFX7-NEXT: s_endpgm
13734 ;
13735 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13736 ; GFX10-WGP: ; %bb.0: ; %entry
13737 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
13738 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13739 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
13740 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
13741 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
13742 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13743 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
13744 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
13745 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
13746 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
13747 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
13748 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
13749 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13750 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
13751 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
13752 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
13753 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13754 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
13755 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
13756 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
13757 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13758 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13759 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13760 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13761 ; GFX10-WGP-NEXT: buffer_gl0_inv
13762 ; GFX10-WGP-NEXT: s_endpgm
13763 ;
13764 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13765 ; GFX10-CU: ; %bb.0: ; %entry
13766 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
13767 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13768 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
13769 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
13770 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
13771 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13772 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
13773 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
13774 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
13775 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
13776 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
13777 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
13778 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13779 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
13780 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
13781 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
13782 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13783 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
13784 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
13785 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
13786 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13787 ; GFX10-CU-NEXT: s_endpgm
13788 ;
13789 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13790 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13791 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
13792 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
13793 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
13794 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
13795 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
13796 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13797 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
13798 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
13799 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
13800 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
13801 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
13802 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
13803 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
13804 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
13805 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
13806 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
13807 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13808 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
13809 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
13810 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
13811 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13812 ; SKIP-CACHE-INV-NEXT: s_endpgm
13813 ;
13814 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13815 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13816 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13817 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13818 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13819 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13820 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13821 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13822 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13823 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13824 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13825 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13826 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13827 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13828 ;
13829 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13830 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13831 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13832 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13833 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13834 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13835 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13836 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13837 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
13838 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13839 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13840 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
13841 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13842 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13843 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13844 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13845 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13846 ;
13847 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13848 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13849 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13850 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13851 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13852 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13853 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13854 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13855 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13856 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13857 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13858 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13859 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13860 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13861 ;
13862 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13863 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13864 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13865 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13866 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13867 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13868 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13869 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13870 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
13871 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13872 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
13873 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
13874 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13875 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
13876 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13877 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13878 ; GFX940-TGSPLIT-NEXT: s_endpgm
13879 ;
13880 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13881 ; GFX11-WGP: ; %bb.0: ; %entry
13882 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13883 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13884 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13885 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13886 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13887 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
13888 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
13889 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13890 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
13891 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
13892 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
13893 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13894 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13895 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13896 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13897 ; GFX11-WGP-NEXT: buffer_gl0_inv
13898 ; GFX11-WGP-NEXT: s_endpgm
13899 ;
13900 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13901 ; GFX11-CU: ; %bb.0: ; %entry
13902 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13903 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13904 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13905 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13906 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13907 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
13908 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
13909 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13910 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
13911 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
13912 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
13913 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13914 ; GFX11-CU-NEXT: s_endpgm
13915 ;
13916 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13917 ; GFX12-WGP: ; %bb.0: ; %entry
13918 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13919 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13920 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13921 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13922 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13923 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
13924 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
13925 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13926 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
13927 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
13928 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
13929 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
13930 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13931 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13932 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13933 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13934 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
13935 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13936 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13937 ; GFX12-WGP-NEXT: s_endpgm
13938 ;
13939 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
13940 ; GFX12-CU: ; %bb.0: ; %entry
13941 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13942 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13943 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13944 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13945 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13946 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
13947 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
13948 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13949 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
13950 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
13951 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
13952 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
13953 ; GFX12-CU-NEXT: s_endpgm
13954 ptr %out, i32 %in, i32 %old) {
13955 entry:
13956 %gep = getelementptr i32, ptr %out, i32 4
13957 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
13958 ret void
13959 }
13961 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
13962 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
13963 ; GFX7: ; %bb.0: ; %entry
13964 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13965 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13966 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13967 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13968 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13969 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13970 ; GFX7-NEXT: s_mov_b32 s4, s8
13971 ; GFX7-NEXT: s_mov_b32 s5, s9
13972 ; GFX7-NEXT: s_mov_b32 s9, s10
13973 ; GFX7-NEXT: s_mov_b32 s8, s11
13974 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13975 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13976 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13977 ; GFX7-NEXT: s_mov_b32 s5, s8
13978 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13979 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13980 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13981 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13982 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13983 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13984 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13985 ; GFX7-NEXT: s_endpgm
13986 ;
13987 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
13988 ; GFX10-WGP: ; %bb.0: ; %entry
13989 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
13990 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13991 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
13992 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
13993 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
13994 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13995 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
13996 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
13997 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
13998 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
13999 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14000 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14001 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14002 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14003 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14004 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14005 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14006 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14007 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14008 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14009 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14010 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14011 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14012 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14013 ; GFX10-WGP-NEXT: buffer_gl0_inv
14014 ; GFX10-WGP-NEXT: s_endpgm
14015 ;
14016 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14017 ; GFX10-CU: ; %bb.0: ; %entry
14018 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
14019 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14020 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14021 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14022 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14023 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14024 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14025 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14026 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14027 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14028 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14029 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14030 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14031 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14032 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14033 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14034 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14035 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14036 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14037 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14038 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14039 ; GFX10-CU-NEXT: s_endpgm
14040 ;
14041 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14042 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14043 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
14044 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14045 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14046 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14047 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14048 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14049 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14050 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14051 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14052 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14053 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14054 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14055 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14056 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14057 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14058 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14059 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14060 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14061 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14062 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14063 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14064 ; SKIP-CACHE-INV-NEXT: s_endpgm
14065 ;
14066 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14067 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14068 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14069 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14070 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14071 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14072 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14073 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14074 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14075 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14076 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14077 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14078 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14079 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14080 ;
14081 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14082 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14083 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14084 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14085 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14086 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14087 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14088 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14089 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14090 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14091 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14092 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14093 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14094 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14095 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14096 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14097 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14098 ;
14099 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14100 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14101 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14102 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14103 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14104 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14105 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14106 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14107 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14108 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14109 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14110 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14111 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14112 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14113 ;
14114 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14115 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14116 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14117 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14118 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14119 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14120 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14121 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14122 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14123 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14124 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14125 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14126 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14127 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14128 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14129 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14130 ; GFX940-TGSPLIT-NEXT: s_endpgm
14131 ;
14132 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14133 ; GFX11-WGP: ; %bb.0: ; %entry
14134 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14135 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14136 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14137 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14138 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14139 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14140 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14141 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14142 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14143 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14144 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14145 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14146 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14147 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14148 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14149 ; GFX11-WGP-NEXT: buffer_gl0_inv
14150 ; GFX11-WGP-NEXT: s_endpgm
14151 ;
14152 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14153 ; GFX11-CU: ; %bb.0: ; %entry
14154 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14155 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14156 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14157 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14158 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14159 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14160 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14161 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14162 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14163 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14164 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14165 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14166 ; GFX11-CU-NEXT: s_endpgm
14167 ;
14168 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14169 ; GFX12-WGP: ; %bb.0: ; %entry
14170 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14171 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14172 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14173 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14174 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14175 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14176 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14177 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14178 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14179 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14180 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14181 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
14182 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14183 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14184 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14185 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14186 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
14187 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14188 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14189 ; GFX12-WGP-NEXT: s_endpgm
14190 ;
14191 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14192 ; GFX12-CU: ; %bb.0: ; %entry
14193 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14194 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14195 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14196 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14197 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14198 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14199 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14200 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14201 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14202 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14203 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14204 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14205 ; GFX12-CU-NEXT: s_endpgm
14206 ptr %out, i32 %in, i32 %old) {
14207 entry:
14208 %gep = getelementptr i32, ptr %out, i32 4
14209 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
14210 ret void
14211 }
14213 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
14214 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14215 ; GFX7: ; %bb.0: ; %entry
14216 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14217 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14218 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14219 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14220 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14221 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14222 ; GFX7-NEXT: s_mov_b32 s4, s8
14223 ; GFX7-NEXT: s_mov_b32 s5, s9
14224 ; GFX7-NEXT: s_mov_b32 s9, s10
14225 ; GFX7-NEXT: s_mov_b32 s8, s11
14226 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14227 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14228 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14229 ; GFX7-NEXT: s_mov_b32 s5, s8
14230 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14231 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14232 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14233 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14234 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14235 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14236 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14237 ; GFX7-NEXT: s_endpgm
14238 ;
14239 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14240 ; GFX10-WGP: ; %bb.0: ; %entry
14241 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
14242 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14243 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14244 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14245 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14246 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14247 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14248 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14249 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14250 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14251 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14252 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14253 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14254 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14255 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14256 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14257 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14258 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14259 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14260 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14261 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14262 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14263 ; GFX10-WGP-NEXT: buffer_gl0_inv
14264 ; GFX10-WGP-NEXT: s_endpgm
14265 ;
14266 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14267 ; GFX10-CU: ; %bb.0: ; %entry
14268 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
14269 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14270 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14271 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14272 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14273 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14274 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14275 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14276 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14277 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14278 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14279 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14280 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14281 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14282 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14283 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14284 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14285 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14286 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14287 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14288 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14289 ; GFX10-CU-NEXT: s_endpgm
14290 ;
14291 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14292 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14293 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
14294 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14295 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14296 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14297 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14298 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14299 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14300 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14301 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14302 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14303 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14304 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14305 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14306 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14309 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14310 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14311 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14312 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14313 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14314 ; SKIP-CACHE-INV-NEXT: s_endpgm
14315 ;
14316 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14317 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14318 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14319 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14320 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14321 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14322 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14323 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14324 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14325 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14326 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14327 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14328 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14329 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14330 ;
14331 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14332 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14333 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14334 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14335 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14336 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14337 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14338 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14339 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14340 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14341 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14342 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14343 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14344 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14345 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14346 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14347 ;
14348 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14349 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14350 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14351 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14352 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14353 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14354 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14355 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14357 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14358 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14359 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14360 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14361 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14362 ;
14363 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14364 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14365 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14366 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14367 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14368 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14369 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14370 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14371 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14372 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14373 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14374 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14375 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14376 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14377 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14378 ; GFX940-TGSPLIT-NEXT: s_endpgm
14379 ;
14380 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14381 ; GFX11-WGP: ; %bb.0: ; %entry
14382 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14383 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14384 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14385 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14386 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14387 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14388 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14389 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14390 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14391 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14392 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14393 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14394 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14395 ; GFX11-WGP-NEXT: buffer_gl0_inv
14396 ; GFX11-WGP-NEXT: s_endpgm
14397 ;
14398 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14399 ; GFX11-CU: ; %bb.0: ; %entry
14400 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14401 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14402 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14403 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14404 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14405 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14406 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14407 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14408 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14409 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14410 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14411 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14412 ; GFX11-CU-NEXT: s_endpgm
14413 ;
14414 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14415 ; GFX12-WGP: ; %bb.0: ; %entry
14416 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14417 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14418 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14419 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14420 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14421 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14423 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14424 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14425 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14426 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14427 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
14428 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14429 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14430 ; GFX12-WGP-NEXT: s_endpgm
14431 ;
14432 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
14433 ; GFX12-CU: ; %bb.0: ; %entry
14434 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14435 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14436 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14437 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14438 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14439 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14440 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14441 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14442 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14443 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14444 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14445 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14446 ; GFX12-CU-NEXT: s_endpgm
14447 ptr %out, i32 %in, i32 %old) {
14448 entry:
14449 %gep = getelementptr i32, ptr %out, i32 4
14450 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
14451 ret void
14452 }
14454 define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
14455 ; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14456 ; GFX7: ; %bb.0: ; %entry
14457 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14458 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14459 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14460 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14461 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14462 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14463 ; GFX7-NEXT: s_mov_b32 s4, s8
14464 ; GFX7-NEXT: s_mov_b32 s5, s9
14465 ; GFX7-NEXT: s_mov_b32 s9, s10
14466 ; GFX7-NEXT: s_mov_b32 s8, s11
14467 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14468 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14469 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14470 ; GFX7-NEXT: s_mov_b32 s5, s8
14471 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14472 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14473 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14474 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14475 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14476 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14477 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14478 ; GFX7-NEXT: s_endpgm
14479 ;
14480 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14481 ; GFX10-WGP: ; %bb.0: ; %entry
14482 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
14483 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14484 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14485 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14486 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14487 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14488 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14489 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14490 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14491 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14492 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14493 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14494 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14495 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14496 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14497 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14498 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14499 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14500 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14501 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14502 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14503 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14504 ; GFX10-WGP-NEXT: buffer_gl0_inv
14505 ; GFX10-WGP-NEXT: s_endpgm
14506 ;
14507 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14508 ; GFX10-CU: ; %bb.0: ; %entry
14509 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
14510 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14511 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14512 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14513 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14514 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14515 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14516 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14517 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14518 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14519 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14520 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14521 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14522 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14523 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14524 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14525 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14526 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14527 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14528 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14529 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14530 ; GFX10-CU-NEXT: s_endpgm
14531 ;
14532 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14533 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14534 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
14535 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14536 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14537 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14538 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14539 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14540 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14541 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14542 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14543 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14544 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14545 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14546 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14547 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14548 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14549 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14550 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14551 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14552 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14553 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14554 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14555 ; SKIP-CACHE-INV-NEXT: s_endpgm
14556 ;
14557 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14558 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14559 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14560 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14561 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14562 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14563 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14564 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14565 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14566 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14567 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14568 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14569 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14570 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14571 ;
14572 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14573 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14574 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14575 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14576 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14577 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14578 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14579 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14580 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14581 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14582 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14583 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14584 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14585 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14586 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14587 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14588 ;
14589 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14590 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14591 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14592 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14593 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14594 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14595 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14596 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14597 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14598 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14599 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14600 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14601 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14602 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14603 ;
14604 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14605 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14606 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14607 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14608 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14609 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14610 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14611 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14612 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14613 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14614 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14615 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14616 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14617 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14618 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14619 ; GFX940-TGSPLIT-NEXT: s_endpgm
14620 ;
14621 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14622 ; GFX11-WGP: ; %bb.0: ; %entry
14623 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14624 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14625 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14626 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14627 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14628 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14629 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14630 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14631 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14632 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14633 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14634 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14635 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14636 ; GFX11-WGP-NEXT: buffer_gl0_inv
14637 ; GFX11-WGP-NEXT: s_endpgm
14638 ;
14639 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14640 ; GFX11-CU: ; %bb.0: ; %entry
14641 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14642 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14643 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14644 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14645 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14646 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14647 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14648 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14649 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14650 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14651 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14652 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14653 ; GFX11-CU-NEXT: s_endpgm
14654 ;
14655 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14656 ; GFX12-WGP: ; %bb.0: ; %entry
14657 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14658 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14659 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14660 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14661 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14662 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14663 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14664 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14665 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14666 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14667 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14668 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
14669 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14670 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14671 ; GFX12-WGP-NEXT: s_endpgm
14672 ;
14673 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
14674 ; GFX12-CU: ; %bb.0: ; %entry
14675 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14676 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14677 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14678 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14679 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14680 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14681 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14682 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14683 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14684 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14685 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14686 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14687 ; GFX12-CU-NEXT: s_endpgm
14688 ptr %out, i32 %in, i32 %old) {
14689 entry:
14690 %gep = getelementptr i32, ptr %out, i32 4
14691 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
14692 ret void
14693 }
14695 define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
14696 ; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14697 ; GFX7: ; %bb.0: ; %entry
14698 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14699 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14700 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14701 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14702 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14703 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14704 ; GFX7-NEXT: s_mov_b32 s4, s8
14705 ; GFX7-NEXT: s_mov_b32 s5, s9
14706 ; GFX7-NEXT: s_mov_b32 s9, s10
14707 ; GFX7-NEXT: s_mov_b32 s8, s11
14708 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14709 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14710 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14711 ; GFX7-NEXT: s_mov_b32 s5, s8
14712 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14713 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14714 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14715 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14716 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14717 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14718 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14719 ; GFX7-NEXT: s_endpgm
14720 ;
14721 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14722 ; GFX10-WGP: ; %bb.0: ; %entry
14723 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
14724 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14725 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14726 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14727 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14728 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14729 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14730 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14731 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14732 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14733 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14734 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14735 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14736 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14737 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14738 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14739 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14740 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14741 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14742 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14743 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14744 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14745 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14746 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14747 ; GFX10-WGP-NEXT: buffer_gl0_inv
14748 ; GFX10-WGP-NEXT: s_endpgm
14749 ;
14750 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14751 ; GFX10-CU: ; %bb.0: ; %entry
14752 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
14753 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14754 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
14755 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
14756 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
14757 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14758 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
14759 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
14760 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
14761 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
14762 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
14763 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
14764 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14765 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
14766 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
14767 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
14768 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14769 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
14770 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
14771 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
14772 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14773 ; GFX10-CU-NEXT: s_endpgm
14774 ;
14775 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14776 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14777 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
14778 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
14779 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
14780 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
14781 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
14782 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14783 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
14784 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
14785 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
14786 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
14787 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
14788 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
14789 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
14790 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
14791 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
14792 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
14793 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14794 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
14795 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
14796 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
14797 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14798 ; SKIP-CACHE-INV-NEXT: s_endpgm
14799 ;
14800 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14801 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14802 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14803 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14804 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14805 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14806 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14807 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14808 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14809 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14810 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14811 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14812 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14813 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14814 ;
14815 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14816 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14817 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14818 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14819 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14820 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14821 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14822 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14823 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
14824 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14825 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14826 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
14827 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14828 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14829 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14830 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14831 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14832 ;
14833 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14834 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14835 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14836 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14837 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14838 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14839 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14840 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14841 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14842 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14843 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14844 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14845 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14846 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14847 ;
14848 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14849 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14850 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14851 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14852 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14853 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14854 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14855 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14856 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
14857 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14858 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
14859 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
14860 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14861 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
14862 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14863 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14864 ; GFX940-TGSPLIT-NEXT: s_endpgm
14865 ;
14866 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14867 ; GFX11-WGP: ; %bb.0: ; %entry
14868 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14869 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14870 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14871 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14872 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14873 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
14874 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
14875 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14876 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
14877 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
14878 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
14879 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14880 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14881 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14882 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14883 ; GFX11-WGP-NEXT: buffer_gl0_inv
14884 ; GFX11-WGP-NEXT: s_endpgm
14885 ;
14886 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14887 ; GFX11-CU: ; %bb.0: ; %entry
14888 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14889 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14890 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14891 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14892 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14893 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
14894 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
14895 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14896 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
14897 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
14898 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
14899 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14900 ; GFX11-CU-NEXT: s_endpgm
14901 ;
14902 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14903 ; GFX12-WGP: ; %bb.0: ; %entry
14904 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14905 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14906 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14907 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14908 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14909 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
14910 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
14911 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14912 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
14913 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
14914 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
14915 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
14916 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14917 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14918 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14919 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14920 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
14921 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14922 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14923 ; GFX12-WGP-NEXT: s_endpgm
14924 ;
14925 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
14926 ; GFX12-CU: ; %bb.0: ; %entry
14927 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14928 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14929 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14930 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14931 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14932 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
14933 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
14934 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14935 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
14936 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
14937 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
14938 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
14939 ; GFX12-CU-NEXT: s_endpgm
14940 ptr %out, i32 %in, i32 %old) {
14941 entry:
14942 %gep = getelementptr i32, ptr %out, i32 4
14943 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
14944 ret void
14945 }
14947 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
14948 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
14949 ; GFX7: ; %bb.0: ; %entry
14950 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14951 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14952 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14953 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14954 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14955 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14956 ; GFX7-NEXT: s_mov_b32 s4, s8
14957 ; GFX7-NEXT: s_mov_b32 s5, s9
14958 ; GFX7-NEXT: s_mov_b32 s9, s10
14959 ; GFX7-NEXT: s_mov_b32 s8, s11
14960 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14961 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14962 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14963 ; GFX7-NEXT: s_mov_b32 s5, s8
14964 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14965 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14966 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14967 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14968 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14969 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14970 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14971 ; GFX7-NEXT: s_endpgm
14972 ;
14973 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
14974 ; GFX10-WGP: ; %bb.0: ; %entry
14975 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
14976 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14977 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
14978 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
14979 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
14980 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14981 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
14982 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
14983 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
14984 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
14985 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
14986 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
14987 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14988 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
14989 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
14990 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
14991 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14992 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
14993 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
14994 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
14995 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14996 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14997 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14998 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14999 ; GFX10-WGP-NEXT: buffer_gl0_inv
15000 ; GFX10-WGP-NEXT: s_endpgm
15001 ;
15002 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15003 ; GFX10-CU: ; %bb.0: ; %entry
15004 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
15005 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15006 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15007 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15008 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15009 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15010 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15011 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15012 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15013 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15014 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15015 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15016 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15017 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15018 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15019 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15020 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15021 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15022 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15023 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15024 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15025 ; GFX10-CU-NEXT: s_endpgm
15026 ;
15027 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15028 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15029 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
15030 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15031 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15032 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15033 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15034 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15035 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15036 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15037 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15038 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15039 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15040 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15041 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15042 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15043 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15044 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15045 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15046 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15047 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15048 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15049 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15050 ; SKIP-CACHE-INV-NEXT: s_endpgm
15051 ;
15052 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15053 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15054 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15055 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15056 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15057 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15058 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15059 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15060 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15061 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15062 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15063 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15064 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15065 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15066 ;
15067 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15068 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15069 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15070 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15071 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15072 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15073 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15074 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15075 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15076 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15077 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15078 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15079 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15080 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15081 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15082 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15083 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15084 ;
15085 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15086 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15087 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15088 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15089 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15090 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15091 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15092 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15093 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15094 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15095 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15096 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15097 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15098 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15099 ;
15100 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15101 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15102 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15103 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15104 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15105 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15106 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15107 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15108 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15109 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15110 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15111 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15112 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15113 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15114 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15115 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15116 ; GFX940-TGSPLIT-NEXT: s_endpgm
15117 ;
15118 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15119 ; GFX11-WGP: ; %bb.0: ; %entry
15120 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15121 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15122 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15123 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15124 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15125 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15127 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15128 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15129 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15130 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15131 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15132 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15133 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15134 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15135 ; GFX11-WGP-NEXT: buffer_gl0_inv
15136 ; GFX11-WGP-NEXT: s_endpgm
15137 ;
15138 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15139 ; GFX11-CU: ; %bb.0: ; %entry
15140 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15141 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15142 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15143 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15144 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15145 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15146 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15147 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15148 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15149 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15150 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15151 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15152 ; GFX11-CU-NEXT: s_endpgm
15153 ;
15154 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15155 ; GFX12-WGP: ; %bb.0: ; %entry
15156 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15157 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15158 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15159 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15160 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15161 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15162 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15163 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15164 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15166 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15167 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15168 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15169 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15170 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15171 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15172 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
15173 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15174 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15175 ; GFX12-WGP-NEXT: s_endpgm
15176 ;
15177 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
15178 ; GFX12-CU: ; %bb.0: ; %entry
15179 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15180 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15181 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15182 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15183 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15184 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15185 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15186 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15187 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15188 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15189 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15190 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15191 ; GFX12-CU-NEXT: s_endpgm
15192 ptr %out, i32 %in, i32 %old) {
15193 entry:
15194 %gep = getelementptr i32, ptr %out, i32 4
15195 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
15196 ret void
15197 }
15199 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
15200 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15201 ; GFX7: ; %bb.0: ; %entry
15202 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15203 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15204 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15205 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15206 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15207 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15208 ; GFX7-NEXT: s_mov_b32 s4, s8
15209 ; GFX7-NEXT: s_mov_b32 s5, s9
15210 ; GFX7-NEXT: s_mov_b32 s9, s10
15211 ; GFX7-NEXT: s_mov_b32 s8, s11
15212 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15213 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15214 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15215 ; GFX7-NEXT: s_mov_b32 s5, s8
15216 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15217 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15218 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15219 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15220 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15221 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15222 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15223 ; GFX7-NEXT: s_endpgm
15224 ;
15225 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15226 ; GFX10-WGP: ; %bb.0: ; %entry
15227 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
15228 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15229 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15230 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15231 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15232 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15233 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15234 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15235 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15236 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15237 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15238 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15239 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15240 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15241 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15242 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15243 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15244 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15245 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15246 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15247 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15248 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15249 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15250 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15251 ; GFX10-WGP-NEXT: buffer_gl0_inv
15252 ; GFX10-WGP-NEXT: s_endpgm
15253 ;
15254 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15255 ; GFX10-CU: ; %bb.0: ; %entry
15256 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
15257 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15258 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15259 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15260 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15261 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15262 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15263 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15264 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15265 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15266 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15267 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15268 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15269 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15270 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15271 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15272 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15273 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15274 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15275 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15276 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15277 ; GFX10-CU-NEXT: s_endpgm
15278 ;
15279 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15280 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15281 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
15282 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15283 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15284 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15285 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15286 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15287 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15288 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15289 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15290 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15291 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15292 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15293 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15294 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15295 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15296 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15297 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15298 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15299 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15300 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15301 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15302 ; SKIP-CACHE-INV-NEXT: s_endpgm
15303 ;
15304 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15305 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15306 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15307 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15308 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15309 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15310 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15311 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15312 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15313 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15314 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15315 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15316 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15317 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15318 ;
15319 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15320 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15321 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15322 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15323 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15324 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15325 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15326 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15327 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15328 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15329 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15330 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15331 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15332 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15333 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15334 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15335 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15336 ;
15337 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15338 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15339 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15340 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15341 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15342 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15343 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15344 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15345 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15346 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15347 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15348 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15349 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15350 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15351 ;
15352 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15353 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15354 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15355 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15356 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15357 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15358 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15359 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15360 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15361 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15362 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15363 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15364 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15365 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15366 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15367 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15368 ; GFX940-TGSPLIT-NEXT: s_endpgm
15369 ;
15370 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15371 ; GFX11-WGP: ; %bb.0: ; %entry
15372 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15373 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15374 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15375 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15376 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15377 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15378 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15379 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15380 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15381 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15382 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15383 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15384 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15385 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15386 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15387 ; GFX11-WGP-NEXT: buffer_gl0_inv
15388 ; GFX11-WGP-NEXT: s_endpgm
15389 ;
15390 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15391 ; GFX11-CU: ; %bb.0: ; %entry
15392 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15393 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15394 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15395 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15396 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15397 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15398 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15399 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15400 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15401 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15402 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15403 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15404 ; GFX11-CU-NEXT: s_endpgm
15405 ;
15406 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15407 ; GFX12-WGP: ; %bb.0: ; %entry
15408 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15409 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15410 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15411 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15412 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15413 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15414 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15415 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15416 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15417 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15418 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15419 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15420 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15421 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15422 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15423 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15424 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
15425 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15426 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15427 ; GFX12-WGP-NEXT: s_endpgm
15428 ;
15429 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
15430 ; GFX12-CU: ; %bb.0: ; %entry
15431 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15432 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15433 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15434 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15435 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15436 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15437 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15438 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15439 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15440 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15441 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15442 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15443 ; GFX12-CU-NEXT: s_endpgm
15444 ptr %out, i32 %in, i32 %old) {
15445 entry:
15446 %gep = getelementptr i32, ptr %out, i32 4
15447 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
15448 ret void
15449 }
15451 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
15452 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15453 ; GFX7: ; %bb.0: ; %entry
15454 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15455 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15456 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15457 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15458 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15459 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15460 ; GFX7-NEXT: s_mov_b32 s4, s8
15461 ; GFX7-NEXT: s_mov_b32 s5, s9
15462 ; GFX7-NEXT: s_mov_b32 s9, s10
15463 ; GFX7-NEXT: s_mov_b32 s8, s11
15464 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15465 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15466 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15467 ; GFX7-NEXT: s_mov_b32 s5, s8
15468 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15469 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15470 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15471 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15472 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15473 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15474 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15475 ; GFX7-NEXT: s_endpgm
15476 ;
15477 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15478 ; GFX10-WGP: ; %bb.0: ; %entry
15479 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
15480 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15481 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15482 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15483 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15484 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15485 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15486 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15487 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15488 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15489 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15490 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15491 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15492 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15493 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15494 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15495 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15496 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15497 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15498 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15499 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15500 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15501 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15502 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15503 ; GFX10-WGP-NEXT: buffer_gl0_inv
15504 ; GFX10-WGP-NEXT: s_endpgm
15505 ;
15506 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15507 ; GFX10-CU: ; %bb.0: ; %entry
15508 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
15509 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15510 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15511 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15512 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15513 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15514 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15515 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15516 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15517 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15518 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15519 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15520 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15521 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15522 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15523 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15524 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15525 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15526 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15527 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15528 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15529 ; GFX10-CU-NEXT: s_endpgm
15530 ;
15531 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15532 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15533 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
15534 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15535 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15536 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15537 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15538 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15539 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15540 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15541 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15542 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15543 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15544 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15545 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15546 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15547 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15548 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15549 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15550 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15551 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15552 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15553 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15554 ; SKIP-CACHE-INV-NEXT: s_endpgm
15555 ;
15556 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15557 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15558 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15559 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15560 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15561 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15562 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15563 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15564 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15565 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15566 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15567 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15568 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15569 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15570 ;
15571 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15572 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15573 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15574 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15575 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15576 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15577 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15578 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15579 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15580 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15581 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15582 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15583 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15584 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15585 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15586 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15587 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15588 ;
15589 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15590 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15591 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15592 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15593 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15594 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15595 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15596 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15597 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15598 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15599 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15600 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15601 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15602 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15603 ;
15604 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15605 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15606 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15607 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15608 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15609 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15610 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15611 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15612 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15613 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15614 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15615 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15616 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15617 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15618 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15619 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15620 ; GFX940-TGSPLIT-NEXT: s_endpgm
15621 ;
15622 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15623 ; GFX11-WGP: ; %bb.0: ; %entry
15624 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15625 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15626 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15627 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15628 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15629 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15630 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15631 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15632 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15633 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15634 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15635 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15636 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15637 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15638 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15639 ; GFX11-WGP-NEXT: buffer_gl0_inv
15640 ; GFX11-WGP-NEXT: s_endpgm
15641 ;
15642 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15643 ; GFX11-CU: ; %bb.0: ; %entry
15644 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15645 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15646 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15647 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15648 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15649 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15650 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15651 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15652 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15653 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15654 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15655 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15656 ; GFX11-CU-NEXT: s_endpgm
15657 ;
15658 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15659 ; GFX12-WGP: ; %bb.0: ; %entry
15660 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15661 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15662 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15663 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15664 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15665 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15666 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15667 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15668 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15669 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15670 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15671 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15672 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15673 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15674 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15675 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15676 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
15677 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15678 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15679 ; GFX12-WGP-NEXT: s_endpgm
15680 ;
15681 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
15682 ; GFX12-CU: ; %bb.0: ; %entry
15683 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15684 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15685 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15686 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15687 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15688 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15689 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15690 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15691 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15692 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15693 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15694 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15695 ; GFX12-CU-NEXT: s_endpgm
15696 ptr %out, i32 %in, i32 %old) {
15697 entry:
15698 %gep = getelementptr i32, ptr %out, i32 4
15699 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
15700 ret void
15701 }
15703 define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
15704 ; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15705 ; GFX7: ; %bb.0: ; %entry
15706 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15707 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15708 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15709 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15710 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15711 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15712 ; GFX7-NEXT: s_mov_b32 s4, s8
15713 ; GFX7-NEXT: s_mov_b32 s5, s9
15714 ; GFX7-NEXT: s_mov_b32 s9, s10
15715 ; GFX7-NEXT: s_mov_b32 s8, s11
15716 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15717 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15718 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15719 ; GFX7-NEXT: s_mov_b32 s5, s8
15720 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15721 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15722 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15723 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15724 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15725 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15726 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15727 ; GFX7-NEXT: s_endpgm
15728 ;
15729 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15730 ; GFX10-WGP: ; %bb.0: ; %entry
15731 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
15732 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15733 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15734 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15735 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15736 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15737 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15738 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15739 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15740 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15741 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15742 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15743 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15744 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15745 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15746 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15747 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15748 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
15749 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
15750 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
15751 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15752 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15753 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15754 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15755 ; GFX10-WGP-NEXT: buffer_gl0_inv
15756 ; GFX10-WGP-NEXT: s_endpgm
15757 ;
15758 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15759 ; GFX10-CU: ; %bb.0: ; %entry
15760 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
15761 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15762 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
15763 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
15764 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
15765 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15766 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
15767 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
15768 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
15769 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
15770 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
15771 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
15772 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15773 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
15774 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
15775 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
15776 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15777 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
15778 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
15779 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
15780 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15781 ; GFX10-CU-NEXT: s_endpgm
15782 ;
15783 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15784 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15785 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
15786 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
15787 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
15788 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
15789 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
15790 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15791 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
15792 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
15793 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
15794 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
15795 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
15796 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
15797 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
15798 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
15799 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
15800 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
15801 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15802 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
15803 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
15804 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
15805 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15806 ; SKIP-CACHE-INV-NEXT: s_endpgm
15807 ;
15808 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15809 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15810 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15811 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15812 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15813 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15814 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15815 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15816 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15817 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15818 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15819 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15820 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15821 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15822 ;
15823 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15824 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15825 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15826 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15827 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15828 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15829 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15830 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15831 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
15832 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15833 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15834 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
15835 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15836 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15837 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15838 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15839 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15840 ;
15841 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15842 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15843 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15844 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15845 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15846 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15847 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15848 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15849 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15850 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15851 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15852 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15853 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15854 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15855 ;
15856 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15857 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15858 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15859 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15860 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15861 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15862 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15863 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15864 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
15865 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15866 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
15867 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
15868 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15869 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
15870 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15871 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15872 ; GFX940-TGSPLIT-NEXT: s_endpgm
15873 ;
15874 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15875 ; GFX11-WGP: ; %bb.0: ; %entry
15876 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15877 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15878 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15879 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15880 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15881 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
15882 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
15883 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15884 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
15885 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
15886 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
15887 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15888 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15889 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15890 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15891 ; GFX11-WGP-NEXT: buffer_gl0_inv
15892 ; GFX11-WGP-NEXT: s_endpgm
15893 ;
15894 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15895 ; GFX11-CU: ; %bb.0: ; %entry
15896 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15897 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15898 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15899 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15900 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15901 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
15902 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
15903 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15904 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
15905 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
15906 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
15907 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15908 ; GFX11-CU-NEXT: s_endpgm
15909 ;
15910 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15911 ; GFX12-WGP: ; %bb.0: ; %entry
15912 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15913 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15914 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15915 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15916 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15917 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
15918 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
15919 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15920 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
15921 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
15922 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
15923 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15924 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15925 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15926 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15927 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15928 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
15929 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15930 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15931 ; GFX12-WGP-NEXT: s_endpgm
15932 ;
15933 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
15934 ; GFX12-CU: ; %bb.0: ; %entry
15935 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15936 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15937 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15938 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15939 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15940 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
15941 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
15942 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15943 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
15944 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
15945 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
15946 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
15947 ; GFX12-CU-NEXT: s_endpgm
15948 ptr %out, i32 %in, i32 %old) {
15949 entry:
15950 %gep = getelementptr i32, ptr %out, i32 4
15951 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
15952 ret void
15953 }
15955 define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
15956 ; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
15957 ; GFX7: ; %bb.0: ; %entry
15958 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15959 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15960 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15961 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15962 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15963 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15964 ; GFX7-NEXT: s_mov_b32 s4, s8
15965 ; GFX7-NEXT: s_mov_b32 s5, s9
15966 ; GFX7-NEXT: s_mov_b32 s9, s10
15967 ; GFX7-NEXT: s_mov_b32 s8, s11
15968 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15969 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15970 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15971 ; GFX7-NEXT: s_mov_b32 s5, s8
15972 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15973 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15974 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15975 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15976 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15977 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15978 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15979 ; GFX7-NEXT: s_endpgm
15980 ;
15981 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
15982 ; GFX10-WGP: ; %bb.0: ; %entry
15983 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
15984 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15985 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
15986 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
15987 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
15988 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15989 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
15990 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
15991 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
15992 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
15993 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
15994 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
15995 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15996 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
15997 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
15998 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
15999 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16000 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16001 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16002 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16003 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16004 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16005 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16006 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16007 ; GFX10-WGP-NEXT: buffer_gl0_inv
16008 ; GFX10-WGP-NEXT: s_endpgm
16009 ;
16010 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16011 ; GFX10-CU: ; %bb.0: ; %entry
16012 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
16013 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16014 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16015 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16016 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16017 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16018 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16019 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16020 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16021 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16022 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16023 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16024 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16025 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16026 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16027 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16028 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16029 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16030 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16031 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16032 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16033 ; GFX10-CU-NEXT: s_endpgm
16034 ;
16035 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16036 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16037 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
16038 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16039 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16040 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16041 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16042 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16043 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16044 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16045 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16046 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16047 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16048 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16049 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16050 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16051 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16052 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16053 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16054 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16055 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16056 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16057 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16058 ; SKIP-CACHE-INV-NEXT: s_endpgm
16059 ;
16060 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16061 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16062 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16063 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16064 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16065 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16066 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16067 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16068 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16069 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16070 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16071 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16072 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16073 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16074 ;
16075 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16076 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16077 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16078 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16079 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16080 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16081 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16082 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16083 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16084 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16085 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16086 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16087 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16088 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16089 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16090 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16091 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16092 ;
16093 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16094 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16095 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16096 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16097 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16098 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16099 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16100 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16101 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16102 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16103 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16104 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16105 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16106 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16107 ;
16108 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16109 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16110 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16111 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16112 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16113 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16114 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16115 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16116 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16117 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16118 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16119 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16120 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16121 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16122 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16123 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16124 ; GFX940-TGSPLIT-NEXT: s_endpgm
16125 ;
16126 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16127 ; GFX11-WGP: ; %bb.0: ; %entry
16128 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16129 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16130 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16131 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16132 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16133 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16134 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16135 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16136 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16137 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16138 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16139 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16140 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16141 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16142 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16143 ; GFX11-WGP-NEXT: buffer_gl0_inv
16144 ; GFX11-WGP-NEXT: s_endpgm
16145 ;
16146 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16147 ; GFX11-CU: ; %bb.0: ; %entry
16148 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16149 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16150 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16151 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16152 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16153 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16154 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16155 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16156 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16157 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16158 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16159 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16160 ; GFX11-CU-NEXT: s_endpgm
16161 ;
16162 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16163 ; GFX12-WGP: ; %bb.0: ; %entry
16164 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16165 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16166 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16167 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16168 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16169 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16170 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16171 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16172 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16173 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16174 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16175 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16176 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16177 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16178 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16179 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16180 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
16181 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16182 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16183 ; GFX12-WGP-NEXT: s_endpgm
16184 ;
16185 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
16186 ; GFX12-CU: ; %bb.0: ; %entry
16187 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16188 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16189 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16190 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16191 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16192 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16193 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16194 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16195 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16196 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16197 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16198 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16199 ; GFX12-CU-NEXT: s_endpgm
16200 ptr %out, i32 %in, i32 %old) {
16201 entry:
16202 %gep = getelementptr i32, ptr %out, i32 4
16203 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
16204 ret void
16205 }
16207 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
16208 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16209 ; GFX7: ; %bb.0: ; %entry
16210 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16211 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16212 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16213 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16214 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16215 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16216 ; GFX7-NEXT: s_mov_b32 s4, s8
16217 ; GFX7-NEXT: s_mov_b32 s5, s9
16218 ; GFX7-NEXT: s_mov_b32 s9, s10
16219 ; GFX7-NEXT: s_mov_b32 s8, s11
16220 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16221 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16222 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16223 ; GFX7-NEXT: s_mov_b32 s5, s8
16224 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16225 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16226 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16227 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16228 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16229 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16230 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16231 ; GFX7-NEXT: s_endpgm
16232 ;
16233 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16234 ; GFX10-WGP: ; %bb.0: ; %entry
16235 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
16236 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16237 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
16238 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
16239 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
16240 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16241 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
16242 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
16243 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
16244 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
16245 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
16246 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
16247 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16248 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
16249 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
16250 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16251 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16252 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16253 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16254 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16255 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16256 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16257 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16258 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16259 ; GFX10-WGP-NEXT: buffer_gl0_inv
16260 ; GFX10-WGP-NEXT: s_endpgm
16261 ;
16262 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16263 ; GFX10-CU: ; %bb.0: ; %entry
16264 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
16265 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16266 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16267 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16268 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16269 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16270 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16271 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16272 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16273 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16274 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16275 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16276 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16277 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16278 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16279 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16280 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16281 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16282 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16283 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16284 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16285 ; GFX10-CU-NEXT: s_endpgm
16286 ;
16287 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16288 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16289 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
16290 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16291 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16292 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16293 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16294 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16296 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16297 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16298 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16299 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16300 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16301 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16302 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16304 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16305 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16309 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16310 ; SKIP-CACHE-INV-NEXT: s_endpgm
16311 ;
16312 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16313 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16314 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16315 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16316 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16317 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16318 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16319 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16320 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16321 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16322 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16323 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16324 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16325 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16326 ;
16327 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16328 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16329 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16330 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16331 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16332 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16333 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16334 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16335 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16336 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16337 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16338 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16339 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16340 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16341 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16342 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16343 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16344 ;
16345 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16346 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16347 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16348 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16349 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16350 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16351 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16352 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16353 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16354 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16355 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16357 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16358 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16359 ;
16360 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16361 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16362 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16363 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16364 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16365 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16366 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16367 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16368 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16369 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16370 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16371 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16372 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16373 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16374 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16375 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16376 ; GFX940-TGSPLIT-NEXT: s_endpgm
16377 ;
16378 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16379 ; GFX11-WGP: ; %bb.0: ; %entry
16380 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16381 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16382 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16383 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16384 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16385 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16386 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16387 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16388 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16389 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16390 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16391 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16392 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16393 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16394 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16395 ; GFX11-WGP-NEXT: buffer_gl0_inv
16396 ; GFX11-WGP-NEXT: s_endpgm
16397 ;
16398 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16399 ; GFX11-CU: ; %bb.0: ; %entry
16400 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16401 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16402 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16403 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16404 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16405 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16406 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16407 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16408 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16409 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16410 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16411 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16412 ; GFX11-CU-NEXT: s_endpgm
16413 ;
16414 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16415 ; GFX12-WGP: ; %bb.0: ; %entry
16416 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16417 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16418 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16419 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16420 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16421 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16423 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16424 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16425 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16426 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16427 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16428 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16429 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16430 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16431 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16432 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
16433 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16434 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16435 ; GFX12-WGP-NEXT: s_endpgm
16436 ;
16437 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16438 ; GFX12-CU: ; %bb.0: ; %entry
16439 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16440 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16441 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16442 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16443 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16444 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16445 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16446 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16447 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16448 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16449 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16450 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16451 ; GFX12-CU-NEXT: s_endpgm
16452 ptr %out, i32 %in, i32 %old) {
16453 entry:
16454 %gep = getelementptr i32, ptr %out, i32 4
16455 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
16456 ret void
16457 }
16459 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
16460 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16461 ; GFX7: ; %bb.0: ; %entry
16462 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16463 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16464 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16465 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16466 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16467 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16468 ; GFX7-NEXT: s_mov_b32 s4, s8
16469 ; GFX7-NEXT: s_mov_b32 s5, s9
16470 ; GFX7-NEXT: s_mov_b32 s9, s10
16471 ; GFX7-NEXT: s_mov_b32 s8, s11
16472 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16473 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16474 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16475 ; GFX7-NEXT: s_mov_b32 s5, s8
16476 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16477 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16478 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16479 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16480 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16481 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16482 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16483 ; GFX7-NEXT: s_endpgm
16484 ;
16485 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16486 ; GFX10-WGP: ; %bb.0: ; %entry
16487 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
16488 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16489 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x8
16490 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0xc
16491 ; GFX10-WGP-NEXT: s_mov_b64 s[10:11], 16
16492 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16493 ; GFX10-WGP-NEXT: s_mov_b32 s4, s8
16494 ; GFX10-WGP-NEXT: s_mov_b32 s5, s9
16495 ; GFX10-WGP-NEXT: s_mov_b32 s9, s10
16496 ; GFX10-WGP-NEXT: s_mov_b32 s8, s11
16497 ; GFX10-WGP-NEXT: s_add_u32 s4, s4, s9
16498 ; GFX10-WGP-NEXT: s_addc_u32 s8, s5, s8
16499 ; GFX10-WGP-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16500 ; GFX10-WGP-NEXT: s_mov_b32 s5, s8
16501 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s7
16502 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16503 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16504 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16505 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16506 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16507 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16508 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16509 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16510 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16511 ; GFX10-WGP-NEXT: buffer_gl0_inv
16512 ; GFX10-WGP-NEXT: s_endpgm
16513 ;
16514 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16515 ; GFX10-CU: ; %bb.0: ; %entry
16516 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
16517 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16518 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x8
16519 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0xc
16520 ; GFX10-CU-NEXT: s_mov_b64 s[10:11], 16
16521 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16522 ; GFX10-CU-NEXT: s_mov_b32 s4, s8
16523 ; GFX10-CU-NEXT: s_mov_b32 s5, s9
16524 ; GFX10-CU-NEXT: s_mov_b32 s9, s10
16525 ; GFX10-CU-NEXT: s_mov_b32 s8, s11
16526 ; GFX10-CU-NEXT: s_add_u32 s4, s4, s9
16527 ; GFX10-CU-NEXT: s_addc_u32 s8, s5, s8
16528 ; GFX10-CU-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16529 ; GFX10-CU-NEXT: s_mov_b32 s5, s8
16530 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s7
16531 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16532 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16533 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16534 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16535 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16536 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16537 ; GFX10-CU-NEXT: s_endpgm
16538 ;
16539 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16540 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16541 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
16542 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
16543 ; SKIP-CACHE-INV-NEXT: s_load_dword s3, s[0:1], 0x2
16544 ; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[0:1], 0x3
16545 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[6:7], 16
16546 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16547 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s4
16548 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s5
16549 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s6
16550 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s7
16551 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, s5
16552 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s4, s1, s4
16553 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
16554 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s4
16555 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s3
16556 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16557 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16559 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16560 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16561 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16562 ; SKIP-CACHE-INV-NEXT: s_endpgm
16563 ;
16564 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16565 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16566 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16567 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16568 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16569 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16570 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16571 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16572 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16573 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16574 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16575 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16576 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16577 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16578 ;
16579 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16580 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16581 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16582 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16583 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16584 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16585 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16586 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16587 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16588 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16589 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16590 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16591 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16592 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16593 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16594 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16595 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16596 ;
16597 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16598 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16599 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16600 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16601 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16602 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16603 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16604 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16605 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16606 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16607 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16608 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16609 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16610 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16611 ;
16612 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16613 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16614 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16615 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16616 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16617 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16618 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16619 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16620 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16621 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16622 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16623 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16624 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16625 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
16626 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16627 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16628 ; GFX940-TGSPLIT-NEXT: s_endpgm
16629 ;
16630 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16631 ; GFX11-WGP: ; %bb.0: ; %entry
16632 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16633 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16634 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16635 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16636 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16637 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16638 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16639 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16640 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16641 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16642 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16643 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16644 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16645 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16646 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16647 ; GFX11-WGP-NEXT: buffer_gl0_inv
16648 ; GFX11-WGP-NEXT: s_endpgm
16649 ;
16650 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16651 ; GFX11-CU: ; %bb.0: ; %entry
16652 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16653 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16654 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16655 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16656 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16657 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16658 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16659 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16660 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16661 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16662 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16663 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16664 ; GFX11-CU-NEXT: s_endpgm
16665 ;
16666 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16667 ; GFX12-WGP: ; %bb.0: ; %entry
16668 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16669 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16670 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16671 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16672 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16673 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16674 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16675 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16676 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16677 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16678 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16679 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16680 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16681 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16682 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16683 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16684 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SE
16685 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16686 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16687 ; GFX12-WGP-NEXT: s_endpgm
16688 ;
16689 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
16690 ; GFX12-CU: ; %bb.0: ; %entry
16691 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16692 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16693 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16694 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16695 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16696 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16697 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16698 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16699 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16700 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16701 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16702 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
16703 ; GFX12-CU-NEXT: s_endpgm
16704 ptr %out, i32 %in, i32 %old) {
16705 entry:
16706 %gep = getelementptr i32, ptr %out, i32 4
16707 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
16708 ret void
16709 }
16711 define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
16712 ; GFX7-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16713 ; GFX7: ; %bb.0: ; %entry
16714 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16715 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
16716 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
16717 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
16718 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16719 ; GFX7-NEXT: s_mov_b32 s6, s4
16720 ; GFX7-NEXT: s_mov_b32 s7, s5
16721 ; GFX7-NEXT: s_mov_b32 s11, s12
16722 ; GFX7-NEXT: s_mov_b32 s10, s13
16723 ; GFX7-NEXT: s_add_u32 s6, s6, s11
16724 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
16725 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
16726 ; GFX7-NEXT: s_mov_b32 s7, s10
16727 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
16728 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
16729 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16730 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16731 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16732 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
16733 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16734 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16735 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16736 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16737 ; GFX7-NEXT: flat_store_dword v[0:1], v2
16738 ; GFX7-NEXT: s_endpgm
16739 ;
16740 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16741 ; GFX10-WGP: ; %bb.0: ; %entry
16742 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16743 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
16744 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
16745 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
16746 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16747 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
16748 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
16749 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
16750 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
16751 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
16752 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
16753 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
16754 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
16755 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
16756 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
16757 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16758 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
16759 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
16760 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
16761 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16762 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
16763 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
16764 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16765 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
16766 ; GFX10-WGP-NEXT: s_endpgm
16767 ;
16768 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16769 ; GFX10-CU: ; %bb.0: ; %entry
16770 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16771 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
16772 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
16773 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
16774 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16775 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
16776 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
16777 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
16778 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
16779 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
16780 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
16781 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
16782 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
16783 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
16784 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
16785 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16786 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
16787 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
16788 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16789 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16790 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
16791 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
16792 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16793 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
16794 ; GFX10-CU-NEXT: s_endpgm
16795 ;
16796 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16797 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16798 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
16799 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
16800 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
16801 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
16802 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16803 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
16804 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
16805 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
16806 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
16807 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
16808 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
16809 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
16810 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
16811 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
16812 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
16813 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16814 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
16815 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
16816 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
16817 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
16818 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
16819 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
16820 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16821 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
16822 ; SKIP-CACHE-INV-NEXT: s_endpgm
16823 ;
16824 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16825 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16826 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16827 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16828 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16829 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16830 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16831 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16832 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16833 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16834 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16835 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16836 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
16837 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16838 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16839 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
16840 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16841 ;
16842 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16843 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16844 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16845 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16846 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16847 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16848 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16849 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16850 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
16851 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16852 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16853 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16854 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
16855 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
16856 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16857 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
16858 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16859 ;
16860 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16861 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16862 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16863 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16864 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16865 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16866 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16867 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16868 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16869 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16870 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16871 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16872 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
16873 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16874 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16875 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
16876 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16877 ;
16878 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16879 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16880 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16881 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16882 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16883 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16884 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16885 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16886 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
16887 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16888 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
16889 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16890 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
16891 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
16892 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16893 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
16894 ; GFX940-TGSPLIT-NEXT: s_endpgm
16895 ;
16896 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16897 ; GFX11-WGP: ; %bb.0: ; %entry
16898 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16899 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16900 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16901 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16902 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16903 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
16904 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
16905 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16906 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
16907 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16908 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16909 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
16910 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
16911 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
16912 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16913 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
16914 ; GFX11-WGP-NEXT: s_endpgm
16915 ;
16916 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16917 ; GFX11-CU: ; %bb.0: ; %entry
16918 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16919 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16920 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16921 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16922 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16923 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
16924 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
16925 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16926 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
16927 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16928 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16929 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
16930 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
16931 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
16932 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
16933 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
16934 ; GFX11-CU-NEXT: s_endpgm
16935 ;
16936 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16937 ; GFX12-WGP: ; %bb.0: ; %entry
16938 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16939 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16940 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16941 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16942 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16943 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
16944 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
16945 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16946 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
16947 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16948 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16949 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
16950 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
16951 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
16952 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
16953 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
16954 ; GFX12-WGP-NEXT: s_endpgm
16955 ;
16956 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
16957 ; GFX12-CU: ; %bb.0: ; %entry
16958 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16959 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16960 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16961 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16962 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16963 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
16964 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
16965 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16966 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
16967 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16968 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16969 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
16970 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
16971 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
16972 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
16973 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
16974 ; GFX12-CU-NEXT: s_endpgm
16975 ptr %out, i32 %in, i32 %old) {
16976 entry:
16977 %gep = getelementptr i32, ptr %out, i32 4
16978 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
16979 %val0 = extractvalue { i32, i1 } %val, 0
16980 store i32 %val0, ptr %out, align 4
16981 ret void
16982 }
16984 define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
16985 ; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
16986 ; GFX7: ; %bb.0: ; %entry
16987 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16988 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
16989 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
16990 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
16991 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16992 ; GFX7-NEXT: s_mov_b32 s6, s4
16993 ; GFX7-NEXT: s_mov_b32 s7, s5
16994 ; GFX7-NEXT: s_mov_b32 s11, s12
16995 ; GFX7-NEXT: s_mov_b32 s10, s13
16996 ; GFX7-NEXT: s_add_u32 s6, s6, s11
16997 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
16998 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
16999 ; GFX7-NEXT: s_mov_b32 s7, s10
17000 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17001 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17002 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17003 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17004 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17005 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17006 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17007 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17008 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17009 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17010 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17011 ; GFX7-NEXT: s_endpgm
17012 ;
17013 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17014 ; GFX10-WGP: ; %bb.0: ; %entry
17015 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17016 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
17017 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
17018 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
17019 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17020 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
17021 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
17022 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
17023 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
17024 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
17025 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
17026 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17027 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
17028 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
17029 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
17030 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17031 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17032 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17033 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17034 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17035 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17036 ; GFX10-WGP-NEXT: buffer_gl0_inv
17037 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17038 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17039 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17040 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
17041 ; GFX10-WGP-NEXT: s_endpgm
17042 ;
17043 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17044 ; GFX10-CU: ; %bb.0: ; %entry
17045 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17046 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
17047 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
17048 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
17049 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17050 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
17051 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
17052 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
17053 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
17054 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
17055 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
17056 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17057 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
17058 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
17059 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
17060 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17061 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17062 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17063 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17064 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17065 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17066 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17067 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17068 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
17069 ; GFX10-CU-NEXT: s_endpgm
17070 ;
17071 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17072 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17073 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17074 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17075 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17076 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
17077 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17078 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
17079 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
17080 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
17081 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
17082 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
17083 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
17084 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
17085 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17086 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
17087 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
17088 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17089 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17090 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17091 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
17092 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17093 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17094 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17095 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17096 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
17097 ; SKIP-CACHE-INV-NEXT: s_endpgm
17098 ;
17099 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17100 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17101 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17102 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17103 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17104 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17105 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17106 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17107 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17108 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17109 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17110 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17111 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17112 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17113 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17114 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
17115 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17116 ;
17117 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17118 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17119 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17120 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17121 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17122 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17123 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17124 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17125 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17126 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17127 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17128 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17129 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17130 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17131 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17132 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17133 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
17134 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17135 ;
17136 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17137 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17138 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17139 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17140 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17141 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17142 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17143 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17144 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17145 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17146 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17147 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17148 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17149 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17150 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17151 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17152 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17153 ;
17154 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17155 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17156 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17157 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17158 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17159 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17160 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17161 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17162 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17163 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17164 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17165 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17166 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17167 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17168 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
17169 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17170 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17171 ; GFX940-TGSPLIT-NEXT: s_endpgm
17172 ;
17173 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17174 ; GFX11-WGP: ; %bb.0: ; %entry
17175 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17176 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17177 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17178 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17179 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17180 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17181 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17182 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17183 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17184 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17185 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17186 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17187 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17188 ; GFX11-WGP-NEXT: buffer_gl0_inv
17189 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17190 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17191 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17192 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
17193 ; GFX11-WGP-NEXT: s_endpgm
17194 ;
17195 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17196 ; GFX11-CU: ; %bb.0: ; %entry
17197 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17198 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17199 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17200 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17201 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17202 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17203 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17204 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17205 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17206 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17207 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17208 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17209 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17210 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17211 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17212 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
17213 ; GFX11-CU-NEXT: s_endpgm
17214 ;
17215 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17216 ; GFX12-WGP: ; %bb.0: ; %entry
17217 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17218 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17219 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17220 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17221 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17222 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17223 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17224 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17225 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17226 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17227 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17228 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
17229 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17230 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17231 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17232 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
17233 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17234 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17235 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
17236 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
17237 ; GFX12-WGP-NEXT: s_endpgm
17238 ;
17239 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17240 ; GFX12-CU: ; %bb.0: ; %entry
17241 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17242 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17243 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17244 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17245 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17246 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17247 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17248 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17249 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17250 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17251 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17252 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
17253 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17254 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17255 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
17256 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
17257 ; GFX12-CU-NEXT: s_endpgm
17258 ptr %out, i32 %in, i32 %old) {
17259 entry:
17260 %gep = getelementptr i32, ptr %out, i32 4
17261 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
17262 %val0 = extractvalue { i32, i1 } %val, 0
17263 store i32 %val0, ptr %out, align 4
17264 ret void
17265 }
17267 define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
17268 ; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17269 ; GFX7: ; %bb.0: ; %entry
17270 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17271 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17272 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17273 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17274 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17275 ; GFX7-NEXT: s_mov_b32 s6, s4
17276 ; GFX7-NEXT: s_mov_b32 s7, s5
17277 ; GFX7-NEXT: s_mov_b32 s11, s12
17278 ; GFX7-NEXT: s_mov_b32 s10, s13
17279 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17280 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17281 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17282 ; GFX7-NEXT: s_mov_b32 s7, s10
17283 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17284 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17285 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17286 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17287 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17288 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17289 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17290 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17291 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17292 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17293 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17294 ; GFX7-NEXT: s_endpgm
17295 ;
17296 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17297 ; GFX10-WGP: ; %bb.0: ; %entry
17298 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17299 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
17300 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
17301 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
17302 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17303 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
17304 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
17305 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
17306 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
17307 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
17308 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
17309 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17310 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
17311 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
17312 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
17313 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17314 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17315 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17316 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17317 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17318 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17319 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17320 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17321 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17322 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17323 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
17324 ; GFX10-WGP-NEXT: s_endpgm
17325 ;
17326 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17327 ; GFX10-CU: ; %bb.0: ; %entry
17328 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17329 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
17330 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
17331 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
17332 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17333 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
17334 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
17335 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
17336 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
17337 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
17338 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
17339 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17340 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
17341 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
17342 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
17343 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17344 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17345 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17346 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17347 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17348 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17349 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17350 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17351 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
17352 ; GFX10-CU-NEXT: s_endpgm
17353 ;
17354 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17355 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17356 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17357 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17358 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17359 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
17360 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17361 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
17362 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
17363 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
17364 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
17365 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
17366 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
17367 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
17368 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17369 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
17370 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
17371 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17372 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17373 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17374 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
17375 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17376 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17377 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17378 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17379 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
17380 ; SKIP-CACHE-INV-NEXT: s_endpgm
17381 ;
17382 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17383 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17384 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17385 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17386 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17387 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17388 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17389 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17390 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17391 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17392 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17393 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17394 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17395 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17396 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17397 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
17398 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17399 ;
17400 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17401 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17402 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17403 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17404 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17405 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17406 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17407 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17408 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17409 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17410 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17411 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17412 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17413 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17414 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17415 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17416 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
17417 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17418 ;
17419 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17420 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17421 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17422 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17423 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17424 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17425 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17426 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17427 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17428 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17429 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17430 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17431 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17432 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17433 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17434 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17435 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17436 ;
17437 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17438 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17439 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17440 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17441 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17442 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17443 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17444 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17445 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17446 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17447 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17448 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17449 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17450 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17451 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17452 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17453 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17454 ; GFX940-TGSPLIT-NEXT: s_endpgm
17455 ;
17456 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17457 ; GFX11-WGP: ; %bb.0: ; %entry
17458 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17459 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17460 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17461 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17462 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17463 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17464 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17465 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17466 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17467 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17468 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17469 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17470 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17471 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17472 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17473 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17474 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17475 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
17476 ; GFX11-WGP-NEXT: s_endpgm
17477 ;
17478 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17479 ; GFX11-CU: ; %bb.0: ; %entry
17480 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17481 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17482 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17483 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17484 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17485 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17486 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17487 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17488 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17489 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17490 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17491 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17492 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17493 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17494 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17495 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
17496 ; GFX11-CU-NEXT: s_endpgm
17497 ;
17498 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17499 ; GFX12-WGP: ; %bb.0: ; %entry
17500 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17501 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17502 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17503 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17504 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17505 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17506 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17507 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17508 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17509 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17510 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17511 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
17512 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17513 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17514 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17515 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17516 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
17517 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17518 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17519 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
17520 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
17521 ; GFX12-WGP-NEXT: s_endpgm
17522 ;
17523 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
17524 ; GFX12-CU: ; %bb.0: ; %entry
17525 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17526 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17527 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17528 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17529 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17530 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17531 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17532 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17533 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17534 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17535 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17536 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
17537 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17538 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17539 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
17540 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
17541 ; GFX12-CU-NEXT: s_endpgm
17542 ptr %out, i32 %in, i32 %old) {
17543 entry:
17544 %gep = getelementptr i32, ptr %out, i32 4
17545 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
17546 %val0 = extractvalue { i32, i1 } %val, 0
17547 store i32 %val0, ptr %out, align 4
17548 ret void
17549 }
17551 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
17552 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17553 ; GFX7: ; %bb.0: ; %entry
17554 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17555 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17556 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17557 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17558 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17559 ; GFX7-NEXT: s_mov_b32 s6, s4
17560 ; GFX7-NEXT: s_mov_b32 s7, s5
17561 ; GFX7-NEXT: s_mov_b32 s11, s12
17562 ; GFX7-NEXT: s_mov_b32 s10, s13
17563 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17564 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17565 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17566 ; GFX7-NEXT: s_mov_b32 s7, s10
17567 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17568 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17569 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17570 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17571 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17572 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17573 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17574 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17575 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17576 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17577 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17578 ; GFX7-NEXT: s_endpgm
17579 ;
17580 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17581 ; GFX10-WGP: ; %bb.0: ; %entry
17582 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17583 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
17584 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
17585 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
17586 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17587 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
17588 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
17589 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
17590 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
17591 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
17592 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
17593 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17594 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
17595 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
17596 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
17597 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17598 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17599 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17600 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17601 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17602 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17603 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17604 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17605 ; GFX10-WGP-NEXT: buffer_gl0_inv
17606 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17607 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17608 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17609 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
17610 ; GFX10-WGP-NEXT: s_endpgm
17611 ;
17612 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17613 ; GFX10-CU: ; %bb.0: ; %entry
17614 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17615 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
17616 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
17617 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
17618 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17619 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
17620 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
17621 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
17622 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
17623 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
17624 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
17625 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17626 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
17627 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
17628 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
17629 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17630 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17631 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17632 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17633 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17634 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17635 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17636 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17637 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
17638 ; GFX10-CU-NEXT: s_endpgm
17639 ;
17640 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17641 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17642 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17643 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17644 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17645 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
17646 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17647 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
17648 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
17649 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
17650 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
17651 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
17652 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
17653 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
17654 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17655 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
17656 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
17657 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17658 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17659 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17660 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
17661 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17662 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17663 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17664 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17665 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
17666 ; SKIP-CACHE-INV-NEXT: s_endpgm
17667 ;
17668 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17669 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17670 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17671 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17672 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17673 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17674 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17675 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17676 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17677 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17678 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17679 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17680 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17681 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17682 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17683 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
17684 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17685 ;
17686 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17687 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17688 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17689 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17690 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17691 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17692 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17693 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17694 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17695 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17696 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17697 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17698 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17699 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17700 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17701 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17702 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17703 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
17704 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17705 ;
17706 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17707 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17708 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17709 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17710 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17711 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17712 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17713 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17714 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17715 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17716 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17717 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17718 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17719 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17720 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17721 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17722 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17723 ;
17724 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17725 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17726 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17727 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17728 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17729 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17730 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17731 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17732 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
17733 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17734 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17735 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17736 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17737 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
17738 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17739 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
17740 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
17741 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
17742 ; GFX940-TGSPLIT-NEXT: s_endpgm
17743 ;
17744 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17745 ; GFX11-WGP: ; %bb.0: ; %entry
17746 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17747 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17748 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17749 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17750 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17751 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
17752 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
17753 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17754 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
17755 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17756 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17757 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17758 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17759 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17760 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17761 ; GFX11-WGP-NEXT: buffer_gl0_inv
17762 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
17763 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
17764 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17765 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
17766 ; GFX11-WGP-NEXT: s_endpgm
17767 ;
17768 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17769 ; GFX11-CU: ; %bb.0: ; %entry
17770 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17771 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17772 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17773 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17774 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17775 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
17776 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
17777 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17778 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
17779 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17780 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17781 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
17782 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
17783 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
17784 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17785 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
17786 ; GFX11-CU-NEXT: s_endpgm
17787 ;
17788 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17789 ; GFX12-WGP: ; %bb.0: ; %entry
17790 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17791 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17792 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17793 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17794 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17795 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
17796 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
17797 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17798 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
17799 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17800 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17801 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
17802 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17803 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17804 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17805 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17806 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
17807 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17808 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17809 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17810 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
17811 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
17812 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
17813 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
17814 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
17815 ; GFX12-WGP-NEXT: s_endpgm
17816 ;
17817 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
17818 ; GFX12-CU: ; %bb.0: ; %entry
17819 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17820 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17821 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17822 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17823 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17824 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
17825 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
17826 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17827 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
17828 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17829 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17830 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
17831 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
17832 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
17833 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
17834 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
17835 ; GFX12-CU-NEXT: s_endpgm
17836 ptr %out, i32 %in, i32 %old) {
17837 entry:
17838 %gep = getelementptr i32, ptr %out, i32 4
17839 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
17840 %val0 = extractvalue { i32, i1 } %val, 0
17841 store i32 %val0, ptr %out, align 4
17842 ret void
17843 }
17845 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
17846 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17847 ; GFX7: ; %bb.0: ; %entry
17848 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17849 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17850 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17851 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17852 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17853 ; GFX7-NEXT: s_mov_b32 s6, s4
17854 ; GFX7-NEXT: s_mov_b32 s7, s5
17855 ; GFX7-NEXT: s_mov_b32 s11, s12
17856 ; GFX7-NEXT: s_mov_b32 s10, s13
17857 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17858 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17859 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17860 ; GFX7-NEXT: s_mov_b32 s7, s10
17861 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17862 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17863 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17864 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17865 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17866 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17867 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17868 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17869 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17870 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17871 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17872 ; GFX7-NEXT: s_endpgm
17873 ;
17874 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17875 ; GFX10-WGP: ; %bb.0: ; %entry
17876 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17877 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
17878 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
17879 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
17880 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17881 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
17882 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
17883 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
17884 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
17885 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
17886 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
17887 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17888 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
17889 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
17890 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
17891 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17892 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
17893 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
17894 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17895 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17896 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17897 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17898 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17899 ; GFX10-WGP-NEXT: buffer_gl0_inv
17900 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
17901 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
17902 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17903 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
17904 ; GFX10-WGP-NEXT: s_endpgm
17905 ;
17906 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17907 ; GFX10-CU: ; %bb.0: ; %entry
17908 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17909 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
17910 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
17911 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
17912 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17913 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
17914 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
17915 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
17916 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
17917 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
17918 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
17919 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17920 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
17921 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
17922 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
17923 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17924 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
17925 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
17926 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17927 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17928 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
17929 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
17930 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17931 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
17932 ; GFX10-CU-NEXT: s_endpgm
17933 ;
17934 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17935 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17936 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17937 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17938 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17939 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
17940 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17941 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
17942 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
17943 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
17944 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
17945 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
17946 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
17947 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
17948 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17949 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
17950 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
17951 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17952 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
17953 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
17954 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
17955 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17956 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
17957 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
17958 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17959 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
17960 ; SKIP-CACHE-INV-NEXT: s_endpgm
17961 ;
17962 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17963 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17964 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17965 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17966 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17967 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17968 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17969 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17970 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17971 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17972 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17973 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17974 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17975 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17976 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
17977 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
17978 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17979 ;
17980 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
17981 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17982 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17983 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17984 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17985 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17986 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17987 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17988 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
17989 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17990 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
17991 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17992 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17993 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
17994 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17995 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17996 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
17997 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
17998 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17999 ;
18000 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18001 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18002 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18003 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18004 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18005 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18006 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18007 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18008 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18009 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18010 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18011 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18012 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18013 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18014 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18015 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18016 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18017 ;
18018 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18019 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18020 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18021 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18022 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18023 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18024 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18025 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18026 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18027 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18028 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18029 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18030 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18031 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18032 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18033 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18034 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18035 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18036 ; GFX940-TGSPLIT-NEXT: s_endpgm
18037 ;
18038 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18039 ; GFX11-WGP: ; %bb.0: ; %entry
18040 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18041 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18042 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18043 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18044 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18045 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18046 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18047 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18048 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18049 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18050 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18051 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18052 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18053 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18054 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18055 ; GFX11-WGP-NEXT: buffer_gl0_inv
18056 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18057 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18058 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18059 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18060 ; GFX11-WGP-NEXT: s_endpgm
18061 ;
18062 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18063 ; GFX11-CU: ; %bb.0: ; %entry
18064 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18065 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18066 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18067 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18068 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18069 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18070 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18071 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18072 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18073 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18074 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18075 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18076 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18077 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18078 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18079 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18080 ; GFX11-CU-NEXT: s_endpgm
18081 ;
18082 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18083 ; GFX12-WGP: ; %bb.0: ; %entry
18084 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18085 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18086 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18087 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18088 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18089 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18090 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18091 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18092 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18093 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18094 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18095 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
18096 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18097 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18098 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18099 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18100 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18101 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18102 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18103 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18104 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18105 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18106 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18107 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
18108 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18109 ; GFX12-WGP-NEXT: s_endpgm
18110 ;
18111 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18112 ; GFX12-CU: ; %bb.0: ; %entry
18113 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18114 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18115 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18116 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18117 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18118 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18119 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18120 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18121 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18122 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18123 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18124 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
18125 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18126 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18127 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18128 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18129 ; GFX12-CU-NEXT: s_endpgm
18130 ptr %out, i32 %in, i32 %old) {
18131 entry:
18132 %gep = getelementptr i32, ptr %out, i32 4
18133 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
18134 %val0 = extractvalue { i32, i1 } %val, 0
18135 store i32 %val0, ptr %out, align 4
18136 ret void
18137 }
18139 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
18140 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18141 ; GFX7: ; %bb.0: ; %entry
18142 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18143 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18144 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18145 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18146 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18147 ; GFX7-NEXT: s_mov_b32 s6, s4
18148 ; GFX7-NEXT: s_mov_b32 s7, s5
18149 ; GFX7-NEXT: s_mov_b32 s11, s12
18150 ; GFX7-NEXT: s_mov_b32 s10, s13
18151 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18152 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18153 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18154 ; GFX7-NEXT: s_mov_b32 s7, s10
18155 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18156 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18157 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18158 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18159 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18160 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18161 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18162 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18163 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18164 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18165 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18166 ; GFX7-NEXT: s_endpgm
18167 ;
18168 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18169 ; GFX10-WGP: ; %bb.0: ; %entry
18170 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18171 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18172 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18173 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18174 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18175 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18176 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18177 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18178 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18179 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18180 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18181 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18182 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18183 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18184 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18185 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18186 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18187 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18188 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18189 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18190 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18191 ; GFX10-WGP-NEXT: buffer_gl0_inv
18192 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18193 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18194 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18195 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18196 ; GFX10-WGP-NEXT: s_endpgm
18197 ;
18198 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18199 ; GFX10-CU: ; %bb.0: ; %entry
18200 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18201 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18202 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18203 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18204 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18205 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18206 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18207 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18208 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18209 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18210 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18211 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18212 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18213 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18214 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18215 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18216 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18217 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18218 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18219 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18220 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18221 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18222 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18223 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18224 ; GFX10-CU-NEXT: s_endpgm
18225 ;
18226 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18227 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18228 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18229 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18230 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18231 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18232 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18233 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18234 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18235 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18236 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18237 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18238 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18239 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18240 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18241 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18242 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18243 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18244 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18245 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18246 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18247 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18249 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18250 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18251 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18252 ; SKIP-CACHE-INV-NEXT: s_endpgm
18253 ;
18254 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18255 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18256 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18257 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18258 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18259 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18260 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18261 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18262 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18263 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18264 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18265 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18266 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18267 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18268 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18269 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18270 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18271 ;
18272 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18273 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18274 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18275 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18276 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18277 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18278 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18279 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18280 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18281 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18282 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18283 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18284 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18285 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18286 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18287 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18288 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18289 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18290 ;
18291 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18292 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18293 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18294 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18295 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18296 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18297 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18298 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18299 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18300 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18301 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18302 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18303 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18304 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18305 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18306 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18307 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18308 ;
18309 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18310 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18311 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18312 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18313 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18314 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18315 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18316 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18317 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18318 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18319 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18320 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18321 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18322 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18323 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18324 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18325 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18326 ; GFX940-TGSPLIT-NEXT: s_endpgm
18327 ;
18328 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18329 ; GFX11-WGP: ; %bb.0: ; %entry
18330 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18331 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18332 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18333 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18334 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18335 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18336 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18337 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18338 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18339 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18340 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18341 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18342 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18343 ; GFX11-WGP-NEXT: buffer_gl0_inv
18344 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18345 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18346 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18347 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18348 ; GFX11-WGP-NEXT: s_endpgm
18349 ;
18350 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18351 ; GFX11-CU: ; %bb.0: ; %entry
18352 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18353 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18354 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18355 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18356 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18357 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18358 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18359 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18360 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18361 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18362 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18363 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18364 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18365 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18366 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18367 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18368 ; GFX11-CU-NEXT: s_endpgm
18369 ;
18370 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18371 ; GFX12-WGP: ; %bb.0: ; %entry
18372 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18373 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18374 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18375 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18376 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18377 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18378 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18379 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18380 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18381 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18382 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18383 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18384 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18385 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18386 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18387 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18388 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18389 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18390 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
18391 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18392 ; GFX12-WGP-NEXT: s_endpgm
18393 ;
18394 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18395 ; GFX12-CU: ; %bb.0: ; %entry
18396 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18397 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18398 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18399 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18400 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18401 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18402 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18403 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18404 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18405 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18406 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18407 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
18408 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18409 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18410 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18411 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18412 ; GFX12-CU-NEXT: s_endpgm
18413 ptr %out, i32 %in, i32 %old) {
18414 entry:
18415 %gep = getelementptr i32, ptr %out, i32 4
18416 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
18417 %val0 = extractvalue { i32, i1 } %val, 0
18418 store i32 %val0, ptr %out, align 4
18419 ret void
18420 }
18422 define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
18423 ; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18424 ; GFX7: ; %bb.0: ; %entry
18425 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18426 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18427 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18428 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18429 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18430 ; GFX7-NEXT: s_mov_b32 s6, s4
18431 ; GFX7-NEXT: s_mov_b32 s7, s5
18432 ; GFX7-NEXT: s_mov_b32 s11, s12
18433 ; GFX7-NEXT: s_mov_b32 s10, s13
18434 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18435 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18436 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18437 ; GFX7-NEXT: s_mov_b32 s7, s10
18438 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18439 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18440 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18441 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18442 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18443 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18444 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18445 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18446 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18447 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18448 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18449 ; GFX7-NEXT: s_endpgm
18450 ;
18451 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18452 ; GFX10-WGP: ; %bb.0: ; %entry
18453 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18454 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18455 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18456 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18457 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18458 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18459 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18460 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18461 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18462 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18463 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18464 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18465 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18466 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18467 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18468 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18469 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18470 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18471 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18472 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18473 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18474 ; GFX10-WGP-NEXT: buffer_gl0_inv
18475 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18476 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18477 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18478 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18479 ; GFX10-WGP-NEXT: s_endpgm
18480 ;
18481 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18482 ; GFX10-CU: ; %bb.0: ; %entry
18483 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18484 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18485 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18486 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18487 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18488 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18489 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18490 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18491 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18492 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18493 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18494 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18495 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18496 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18497 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18498 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18499 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18500 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18501 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18502 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18503 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18504 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18505 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18506 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18507 ; GFX10-CU-NEXT: s_endpgm
18508 ;
18509 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18510 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18511 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18512 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18513 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18514 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18515 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18516 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18517 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18518 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18519 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18520 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18521 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18522 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18523 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18524 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18525 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18526 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18527 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18528 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18529 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18530 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18531 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18532 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18533 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18534 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18535 ; SKIP-CACHE-INV-NEXT: s_endpgm
18536 ;
18537 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18538 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18539 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18540 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18541 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18542 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18543 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18544 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18545 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18546 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18547 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18548 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18549 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18550 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18551 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18552 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18553 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18554 ;
18555 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18556 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18557 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18558 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18559 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18560 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18561 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18562 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18563 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18564 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18565 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18566 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18567 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18568 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18569 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18570 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18571 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18572 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18573 ;
18574 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18575 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18576 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18577 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18578 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18579 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18580 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18581 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18582 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18583 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18584 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18585 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18586 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18587 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18588 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18589 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18590 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18591 ;
18592 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18593 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18594 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18595 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18596 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18597 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18598 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18599 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18600 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18601 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18602 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18603 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18604 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18605 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18606 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18607 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18608 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18609 ; GFX940-TGSPLIT-NEXT: s_endpgm
18610 ;
18611 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18612 ; GFX11-WGP: ; %bb.0: ; %entry
18613 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18614 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18615 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18616 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18617 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18618 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18619 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18620 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18621 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18622 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18623 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18624 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18625 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18626 ; GFX11-WGP-NEXT: buffer_gl0_inv
18627 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18628 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18629 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18630 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18631 ; GFX11-WGP-NEXT: s_endpgm
18632 ;
18633 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18634 ; GFX11-CU: ; %bb.0: ; %entry
18635 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18636 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18637 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18638 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18639 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18640 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18641 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18642 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18643 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18644 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18645 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18646 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18647 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18648 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18649 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18650 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18651 ; GFX11-CU-NEXT: s_endpgm
18652 ;
18653 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18654 ; GFX12-WGP: ; %bb.0: ; %entry
18655 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18656 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18657 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18658 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18659 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18660 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18661 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18662 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18663 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18664 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18665 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18666 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18667 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18668 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18669 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18670 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18671 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18672 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18673 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
18674 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18675 ; GFX12-WGP-NEXT: s_endpgm
18676 ;
18677 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
18678 ; GFX12-CU: ; %bb.0: ; %entry
18679 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18680 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18681 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18682 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18683 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18684 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18685 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18686 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18687 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18688 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18689 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18690 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
18691 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18692 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18693 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18694 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18695 ; GFX12-CU-NEXT: s_endpgm
18696 ptr %out, i32 %in, i32 %old) {
18697 entry:
18698 %gep = getelementptr i32, ptr %out, i32 4
18699 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
18700 %val0 = extractvalue { i32, i1 } %val, 0
18701 store i32 %val0, ptr %out, align 4
18702 ret void
18703 }
18705 define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
18706 ; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18707 ; GFX7: ; %bb.0: ; %entry
18708 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18709 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18710 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18711 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18712 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18713 ; GFX7-NEXT: s_mov_b32 s6, s4
18714 ; GFX7-NEXT: s_mov_b32 s7, s5
18715 ; GFX7-NEXT: s_mov_b32 s11, s12
18716 ; GFX7-NEXT: s_mov_b32 s10, s13
18717 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18718 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18719 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18720 ; GFX7-NEXT: s_mov_b32 s7, s10
18721 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18722 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18723 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18724 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18725 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18726 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18727 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18728 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18729 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18730 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18731 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18732 ; GFX7-NEXT: s_endpgm
18733 ;
18734 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18735 ; GFX10-WGP: ; %bb.0: ; %entry
18736 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18737 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
18738 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
18739 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
18740 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18741 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
18742 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
18743 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
18744 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
18745 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
18746 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
18747 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18748 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
18749 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
18750 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
18751 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18752 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
18753 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
18754 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18755 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18756 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18757 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18758 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18759 ; GFX10-WGP-NEXT: buffer_gl0_inv
18760 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
18761 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
18762 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18763 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
18764 ; GFX10-WGP-NEXT: s_endpgm
18765 ;
18766 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18767 ; GFX10-CU: ; %bb.0: ; %entry
18768 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18769 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
18770 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
18771 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
18772 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18773 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
18774 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
18775 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
18776 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
18777 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
18778 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
18779 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18780 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
18781 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
18782 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
18783 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18784 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
18785 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
18786 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18787 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18788 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
18789 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
18790 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18791 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
18792 ; GFX10-CU-NEXT: s_endpgm
18793 ;
18794 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18795 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18796 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18797 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18798 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18799 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
18800 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18801 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
18802 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
18803 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
18804 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
18805 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
18806 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
18807 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
18808 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18809 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
18810 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
18811 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18812 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
18813 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
18814 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
18815 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18816 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
18817 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
18818 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18819 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
18820 ; SKIP-CACHE-INV-NEXT: s_endpgm
18821 ;
18822 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18823 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18824 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18825 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18826 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18827 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18828 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18829 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18830 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18831 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18832 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18833 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18834 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18835 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18836 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18837 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
18838 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18839 ;
18840 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18841 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18842 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18843 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18844 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18845 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18846 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18847 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18848 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
18849 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18850 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18851 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18852 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18853 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
18854 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18855 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18856 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
18857 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
18858 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18859 ;
18860 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18861 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18862 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18863 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18864 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18865 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18866 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18867 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18868 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18869 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18870 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18871 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18872 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18873 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18874 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18875 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18876 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18877 ;
18878 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18879 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18880 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18881 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18882 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18883 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18884 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18885 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18886 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
18887 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18888 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
18889 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18890 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18891 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
18892 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18893 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18894 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
18895 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
18896 ; GFX940-TGSPLIT-NEXT: s_endpgm
18897 ;
18898 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18899 ; GFX11-WGP: ; %bb.0: ; %entry
18900 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18901 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18902 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18903 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18904 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18905 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
18906 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
18907 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18908 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
18909 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18910 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18911 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18912 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18913 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18914 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18915 ; GFX11-WGP-NEXT: buffer_gl0_inv
18916 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
18917 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
18918 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18919 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
18920 ; GFX11-WGP-NEXT: s_endpgm
18921 ;
18922 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18923 ; GFX11-CU: ; %bb.0: ; %entry
18924 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18925 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18926 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18927 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18928 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18929 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
18930 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
18931 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18932 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
18933 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18934 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18935 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
18936 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
18937 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
18938 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
18939 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
18940 ; GFX11-CU-NEXT: s_endpgm
18941 ;
18942 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18943 ; GFX12-WGP: ; %bb.0: ; %entry
18944 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18945 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18946 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18947 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18948 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18949 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
18950 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
18951 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18952 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
18953 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18954 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18955 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
18956 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18957 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18958 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18959 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18960 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18961 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18962 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18963 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18964 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18965 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
18966 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
18967 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
18968 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
18969 ; GFX12-WGP-NEXT: s_endpgm
18970 ;
18971 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
18972 ; GFX12-CU: ; %bb.0: ; %entry
18973 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18974 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18975 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18976 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18977 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18978 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
18979 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
18980 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18981 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
18982 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18983 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18984 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
18985 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
18986 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
18987 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
18988 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
18989 ; GFX12-CU-NEXT: s_endpgm
18990 ptr %out, i32 %in, i32 %old) {
18991 entry:
18992 %gep = getelementptr i32, ptr %out, i32 4
18993 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
18994 %val0 = extractvalue { i32, i1 } %val, 0
18995 store i32 %val0, ptr %out, align 4
18996 ret void
18997 }
18999 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
19000 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19001 ; GFX7: ; %bb.0: ; %entry
19002 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19003 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19004 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19005 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19006 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19007 ; GFX7-NEXT: s_mov_b32 s6, s4
19008 ; GFX7-NEXT: s_mov_b32 s7, s5
19009 ; GFX7-NEXT: s_mov_b32 s11, s12
19010 ; GFX7-NEXT: s_mov_b32 s10, s13
19011 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19012 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19013 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19014 ; GFX7-NEXT: s_mov_b32 s7, s10
19015 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19016 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19017 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19018 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19019 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19020 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19021 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19022 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19023 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19024 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19025 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19026 ; GFX7-NEXT: s_endpgm
19027 ;
19028 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19029 ; GFX10-WGP: ; %bb.0: ; %entry
19030 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19031 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19032 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19033 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19034 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19035 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19036 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19037 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19038 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19039 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19040 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19041 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19042 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19043 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19044 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19045 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19046 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19047 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19048 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19049 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19050 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19051 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19052 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19053 ; GFX10-WGP-NEXT: buffer_gl0_inv
19054 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19055 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19056 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19057 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19058 ; GFX10-WGP-NEXT: s_endpgm
19059 ;
19060 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19061 ; GFX10-CU: ; %bb.0: ; %entry
19062 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19063 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19064 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19065 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19066 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19067 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19068 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19069 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19070 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19071 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19072 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19073 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19074 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19075 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19076 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19077 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19078 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19079 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19080 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19081 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19082 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19083 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19084 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19085 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19086 ; GFX10-CU-NEXT: s_endpgm
19087 ;
19088 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19089 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19090 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19091 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19092 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19093 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19094 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19095 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19096 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19097 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19098 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19099 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19100 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19101 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19102 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19103 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19104 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19105 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19106 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19107 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19108 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19109 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19110 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19111 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19112 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19113 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19114 ; SKIP-CACHE-INV-NEXT: s_endpgm
19115 ;
19116 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19117 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19118 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19119 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19120 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19121 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19122 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19123 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19124 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19125 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19126 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19127 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19128 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19129 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19130 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19131 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19132 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19133 ;
19134 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19135 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19136 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19137 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19138 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19139 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19140 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19141 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19142 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19143 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19144 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19145 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19146 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19147 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19148 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19149 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19150 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19151 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19152 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19153 ;
19154 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19155 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19156 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19157 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19158 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19159 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19160 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19161 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19162 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19163 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19164 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19165 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19166 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19167 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19168 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19169 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19170 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19171 ;
19172 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19173 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19174 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19175 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19176 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19177 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19178 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19179 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19180 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19181 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19182 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19183 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19184 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19185 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19186 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19187 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19188 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19189 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19190 ; GFX940-TGSPLIT-NEXT: s_endpgm
19191 ;
19192 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19193 ; GFX11-WGP: ; %bb.0: ; %entry
19194 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19195 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19196 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19197 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19198 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19199 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19200 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19201 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19202 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19203 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19204 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19205 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19206 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19207 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19208 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19209 ; GFX11-WGP-NEXT: buffer_gl0_inv
19210 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19211 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19212 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19213 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19214 ; GFX11-WGP-NEXT: s_endpgm
19215 ;
19216 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19217 ; GFX11-CU: ; %bb.0: ; %entry
19218 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19219 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19220 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19221 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19222 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19223 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19224 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19225 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19226 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19227 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19228 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19229 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19230 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19231 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19232 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19233 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19234 ; GFX11-CU-NEXT: s_endpgm
19235 ;
19236 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19237 ; GFX12-WGP: ; %bb.0: ; %entry
19238 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19239 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19240 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19241 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19242 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19243 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19244 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19245 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19246 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19247 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19248 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19249 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
19250 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19251 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19252 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19253 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19254 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19255 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19256 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19257 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19258 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19259 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19260 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19261 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19262 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19263 ; GFX12-WGP-NEXT: s_endpgm
19264 ;
19265 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19266 ; GFX12-CU: ; %bb.0: ; %entry
19267 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19268 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19269 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19270 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19271 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19272 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19273 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19274 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19275 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19276 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19277 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19278 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
19279 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19280 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19281 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
19282 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19283 ; GFX12-CU-NEXT: s_endpgm
19284 ptr %out, i32 %in, i32 %old) {
19285 entry:
19286 %gep = getelementptr i32, ptr %out, i32 4
19287 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
19288 %val0 = extractvalue { i32, i1 } %val, 0
19289 store i32 %val0, ptr %out, align 4
19290 ret void
19291 }
19293 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
19294 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19295 ; GFX7: ; %bb.0: ; %entry
19296 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19297 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19298 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19299 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19300 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19301 ; GFX7-NEXT: s_mov_b32 s6, s4
19302 ; GFX7-NEXT: s_mov_b32 s7, s5
19303 ; GFX7-NEXT: s_mov_b32 s11, s12
19304 ; GFX7-NEXT: s_mov_b32 s10, s13
19305 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19306 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19307 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19308 ; GFX7-NEXT: s_mov_b32 s7, s10
19309 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19310 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19311 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19312 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19313 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19314 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19315 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19316 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19317 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19318 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19319 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19320 ; GFX7-NEXT: s_endpgm
19321 ;
19322 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19323 ; GFX10-WGP: ; %bb.0: ; %entry
19324 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19325 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19326 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19327 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19328 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19329 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19330 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19331 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19332 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19333 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19334 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19335 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19336 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19337 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19338 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19339 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19340 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19341 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19342 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19343 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19344 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19345 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19346 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19347 ; GFX10-WGP-NEXT: buffer_gl0_inv
19348 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19349 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19350 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19351 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19352 ; GFX10-WGP-NEXT: s_endpgm
19353 ;
19354 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19355 ; GFX10-CU: ; %bb.0: ; %entry
19356 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19357 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19358 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19359 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19360 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19361 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19362 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19363 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19364 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19365 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19366 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19367 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19368 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19369 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19370 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19371 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19372 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19373 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19374 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19375 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19376 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19377 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19378 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19379 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19380 ; GFX10-CU-NEXT: s_endpgm
19381 ;
19382 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19383 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19384 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19385 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19386 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19387 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19388 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19389 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19390 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19391 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19392 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19393 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19394 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19395 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19396 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19397 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19398 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19399 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19400 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19401 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19402 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19403 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19404 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19405 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19406 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19407 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19408 ; SKIP-CACHE-INV-NEXT: s_endpgm
19409 ;
19410 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19411 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19412 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19413 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19414 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19415 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19416 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19417 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19418 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19419 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19420 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19421 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19422 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19423 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19424 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19425 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19426 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19427 ;
19428 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19429 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19430 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19431 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19432 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19433 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19434 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19435 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19436 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19437 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19438 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19439 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19440 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19441 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19442 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19443 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19444 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19445 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19446 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19447 ;
19448 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19449 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19450 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19451 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19452 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19453 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19454 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19455 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19456 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19457 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19458 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19459 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19460 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19461 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19462 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19463 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19464 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19465 ;
19466 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19467 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19468 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19469 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19470 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19471 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19472 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19473 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19474 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19475 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19476 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19477 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19478 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19479 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19480 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19481 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19482 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19483 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19484 ; GFX940-TGSPLIT-NEXT: s_endpgm
19485 ;
19486 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19487 ; GFX11-WGP: ; %bb.0: ; %entry
19488 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19489 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19490 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19491 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19492 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19493 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19494 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19495 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19496 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19497 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19498 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19499 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19500 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19501 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19502 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19503 ; GFX11-WGP-NEXT: buffer_gl0_inv
19504 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19505 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19506 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19507 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19508 ; GFX11-WGP-NEXT: s_endpgm
19509 ;
19510 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19511 ; GFX11-CU: ; %bb.0: ; %entry
19512 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19513 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19514 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19515 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19516 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19517 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19518 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19519 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19520 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19521 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19522 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19523 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19524 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19525 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19526 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19527 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19528 ; GFX11-CU-NEXT: s_endpgm
19529 ;
19530 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19531 ; GFX12-WGP: ; %bb.0: ; %entry
19532 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19533 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19534 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19535 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19536 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19537 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19538 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19539 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19540 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19541 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19542 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19543 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
19544 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19545 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19546 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19547 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19548 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19549 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19550 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19551 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19552 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19553 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19554 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19555 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19556 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19557 ; GFX12-WGP-NEXT: s_endpgm
19558 ;
19559 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19560 ; GFX12-CU: ; %bb.0: ; %entry
19561 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19562 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19563 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19564 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19565 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19566 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19567 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19568 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19569 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19570 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19571 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19572 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
19573 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19574 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19575 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
19576 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19577 ; GFX12-CU-NEXT: s_endpgm
19578 ptr %out, i32 %in, i32 %old) {
19579 entry:
19580 %gep = getelementptr i32, ptr %out, i32 4
19581 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
19582 %val0 = extractvalue { i32, i1 } %val, 0
19583 store i32 %val0, ptr %out, align 4
19584 ret void
19585 }
19587 define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
19588 ; GFX7-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19589 ; GFX7: ; %bb.0: ; %entry
19590 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19591 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19592 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19593 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19594 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19595 ; GFX7-NEXT: s_mov_b32 s6, s4
19596 ; GFX7-NEXT: s_mov_b32 s7, s5
19597 ; GFX7-NEXT: s_mov_b32 s11, s12
19598 ; GFX7-NEXT: s_mov_b32 s10, s13
19599 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19600 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19601 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19602 ; GFX7-NEXT: s_mov_b32 s7, s10
19603 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19604 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19605 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19606 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19607 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19608 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19609 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19610 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19611 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19612 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19613 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19614 ; GFX7-NEXT: s_endpgm
19615 ;
19616 ; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19617 ; GFX10-WGP: ; %bb.0: ; %entry
19618 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19619 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19620 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19621 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19622 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19623 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19624 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19625 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19626 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19627 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19628 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19629 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19630 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19631 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19632 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19633 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19634 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19635 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19636 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19637 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19638 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19639 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19640 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19641 ; GFX10-WGP-NEXT: buffer_gl0_inv
19642 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19643 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19644 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19645 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19646 ; GFX10-WGP-NEXT: s_endpgm
19647 ;
19648 ; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19649 ; GFX10-CU: ; %bb.0: ; %entry
19650 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19651 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19652 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19653 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19654 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19655 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19656 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19657 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19658 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19659 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19660 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19661 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19662 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19663 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19664 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19665 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19666 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19667 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19668 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19669 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19670 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19671 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19672 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19673 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19674 ; GFX10-CU-NEXT: s_endpgm
19675 ;
19676 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19677 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19678 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19679 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19680 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19681 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19682 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19683 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19684 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19685 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19686 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19687 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19688 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19689 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19690 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19691 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19692 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19693 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19694 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19695 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19696 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19697 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19698 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19699 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19700 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19701 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19702 ; SKIP-CACHE-INV-NEXT: s_endpgm
19703 ;
19704 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19705 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19706 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19707 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19708 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19709 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19710 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19711 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19712 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19713 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19714 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19715 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19716 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19717 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19718 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19719 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
19720 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19721 ;
19722 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19723 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19724 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19725 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19726 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19727 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19728 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19729 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19730 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
19731 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19732 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19733 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19734 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19735 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
19736 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19737 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19738 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
19739 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
19740 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19741 ;
19742 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19743 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19744 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19745 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19746 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19747 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19748 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19749 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19750 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19751 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19752 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19753 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19754 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19755 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19756 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19757 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19758 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19759 ;
19760 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19761 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19762 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19763 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19764 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19765 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19766 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19767 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19768 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
19769 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19770 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
19771 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19772 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19773 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
19774 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19775 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19776 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
19777 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
19778 ; GFX940-TGSPLIT-NEXT: s_endpgm
19779 ;
19780 ; GFX11-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19781 ; GFX11-WGP: ; %bb.0: ; %entry
19782 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19783 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19784 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19785 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19786 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19787 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
19788 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
19789 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19790 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
19791 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19792 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19793 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19794 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19795 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19796 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19797 ; GFX11-WGP-NEXT: buffer_gl0_inv
19798 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
19799 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
19800 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19801 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
19802 ; GFX11-WGP-NEXT: s_endpgm
19803 ;
19804 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19805 ; GFX11-CU: ; %bb.0: ; %entry
19806 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19807 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19808 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19809 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19810 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19811 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
19812 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
19813 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19814 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
19815 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19816 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19817 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
19818 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
19819 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
19820 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19821 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
19822 ; GFX11-CU-NEXT: s_endpgm
19823 ;
19824 ; GFX12-WGP-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19825 ; GFX12-WGP: ; %bb.0: ; %entry
19826 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19827 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19828 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19829 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19830 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19831 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
19832 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
19833 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19834 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
19835 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19836 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19837 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
19838 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19839 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19840 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19841 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19842 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19843 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19844 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19845 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19846 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19847 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
19848 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
19849 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
19850 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
19851 ; GFX12-WGP-NEXT: s_endpgm
19852 ;
19853 ; GFX12-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
19854 ; GFX12-CU: ; %bb.0: ; %entry
19855 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19856 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19857 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19858 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19859 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19860 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
19861 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
19862 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19863 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
19864 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19865 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19866 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
19867 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
19868 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
19869 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
19870 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
19871 ; GFX12-CU-NEXT: s_endpgm
19872 ptr %out, i32 %in, i32 %old) {
19873 entry:
19874 %gep = getelementptr i32, ptr %out, i32 4
19875 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
19876 %val0 = extractvalue { i32, i1 } %val, 0
19877 store i32 %val0, ptr %out, align 4
19878 ret void
19879 }
19881 define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
19882 ; GFX7-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
19883 ; GFX7: ; %bb.0: ; %entry
19884 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19885 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19886 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19887 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19888 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19889 ; GFX7-NEXT: s_mov_b32 s6, s4
19890 ; GFX7-NEXT: s_mov_b32 s7, s5
19891 ; GFX7-NEXT: s_mov_b32 s11, s12
19892 ; GFX7-NEXT: s_mov_b32 s10, s13
19893 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19894 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19895 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19896 ; GFX7-NEXT: s_mov_b32 s7, s10
19897 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19898 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19899 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19900 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19901 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19902 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19903 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19904 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19905 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19906 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19907 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19908 ; GFX7-NEXT: s_endpgm
19909 ;
19910 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
19911 ; GFX10-WGP: ; %bb.0: ; %entry
19912 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19913 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
19914 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
19915 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
19916 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19917 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
19918 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
19919 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
19920 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
19921 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
19922 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
19923 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19924 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
19925 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
19926 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
19927 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19928 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
19929 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
19930 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19931 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19932 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19933 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19934 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19935 ; GFX10-WGP-NEXT: buffer_gl0_inv
19936 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
19937 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
19938 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19939 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
19940 ; GFX10-WGP-NEXT: s_endpgm
19941 ;
19942 ; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
19943 ; GFX10-CU: ; %bb.0: ; %entry
19944 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19945 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
19946 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
19947 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
19948 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19949 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
19950 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
19951 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
19952 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
19953 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
19954 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
19955 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19956 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
19957 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
19958 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
19959 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19960 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
19961 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
19962 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19963 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19964 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
19965 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
19966 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19967 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
19968 ; GFX10-CU-NEXT: s_endpgm
19969 ;
19970 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
19971 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19972 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19973 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19974 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19975 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
19976 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19977 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
19978 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
19979 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
19980 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
19981 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
19982 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
19983 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
19984 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19985 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
19986 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
19987 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19988 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
19989 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
19990 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
19991 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19992 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
19993 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
19994 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
19995 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
19996 ; SKIP-CACHE-INV-NEXT: s_endpgm
19997 ;
19998 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
19999 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20000 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20001 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20002 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20003 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20004 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20005 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20006 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20007 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20008 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20009 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20010 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20011 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20012 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20013 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20014 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20015 ;
20016 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20017 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20018 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20019 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20020 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20021 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20022 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20023 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20024 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20025 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20026 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20027 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20028 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20029 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20030 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20031 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20032 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20033 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20034 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20035 ;
20036 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20037 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20038 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20039 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20040 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20041 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20042 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20043 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20044 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20045 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20046 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20047 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20048 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20049 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20050 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20051 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20052 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20053 ;
20054 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20055 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20056 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20057 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20058 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20059 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20060 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20061 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20062 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20063 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20064 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20065 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20066 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20067 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20068 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20069 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20070 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20071 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20072 ; GFX940-TGSPLIT-NEXT: s_endpgm
20073 ;
20074 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20075 ; GFX11-WGP: ; %bb.0: ; %entry
20076 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20077 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20078 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20079 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20080 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20081 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20082 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20083 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20084 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20085 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20086 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20087 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20088 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20089 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20090 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20091 ; GFX11-WGP-NEXT: buffer_gl0_inv
20092 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20093 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20094 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20095 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20096 ; GFX11-WGP-NEXT: s_endpgm
20097 ;
20098 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20099 ; GFX11-CU: ; %bb.0: ; %entry
20100 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20101 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20102 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20103 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20104 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20105 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20106 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20107 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20108 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20109 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20110 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20111 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20112 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20113 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20114 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20115 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20116 ; GFX11-CU-NEXT: s_endpgm
20117 ;
20118 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20119 ; GFX12-WGP: ; %bb.0: ; %entry
20120 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20121 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20122 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20123 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20124 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20125 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20126 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20127 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20128 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20129 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20130 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20131 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20132 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20133 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20134 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20135 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20136 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20137 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20138 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20139 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20140 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20141 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20142 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20143 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20144 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20145 ; GFX12-WGP-NEXT: s_endpgm
20146 ;
20147 ; GFX12-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20148 ; GFX12-CU: ; %bb.0: ; %entry
20149 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20150 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20151 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20152 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20153 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20154 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20155 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20156 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20157 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20158 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20159 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20160 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
20161 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20162 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20163 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
20164 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20165 ; GFX12-CU-NEXT: s_endpgm
20166 ptr %out, i32 %in, i32 %old) {
20167 entry:
20168 %gep = getelementptr i32, ptr %out, i32 4
20169 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
20170 %val0 = extractvalue { i32, i1 } %val, 0
20171 store i32 %val0, ptr %out, align 4
20172 ret void
20173 }
20175 define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
20176 ; GFX7-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20177 ; GFX7: ; %bb.0: ; %entry
20178 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20179 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20180 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20181 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20182 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20183 ; GFX7-NEXT: s_mov_b32 s6, s4
20184 ; GFX7-NEXT: s_mov_b32 s7, s5
20185 ; GFX7-NEXT: s_mov_b32 s11, s12
20186 ; GFX7-NEXT: s_mov_b32 s10, s13
20187 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20188 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20189 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20190 ; GFX7-NEXT: s_mov_b32 s7, s10
20191 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20192 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20193 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20194 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20195 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20196 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20197 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20198 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20199 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20200 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20201 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20202 ; GFX7-NEXT: s_endpgm
20203 ;
20204 ; GFX10-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20205 ; GFX10-WGP: ; %bb.0: ; %entry
20206 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20207 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20208 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20209 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20210 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20211 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20212 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20213 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20214 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20215 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20216 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20217 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20218 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20219 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20220 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20221 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20222 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20223 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20224 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20225 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20226 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20227 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20228 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20229 ; GFX10-WGP-NEXT: buffer_gl0_inv
20230 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20231 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20232 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20233 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20234 ; GFX10-WGP-NEXT: s_endpgm
20235 ;
20236 ; GFX10-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20237 ; GFX10-CU: ; %bb.0: ; %entry
20238 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20239 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20240 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20241 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20242 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20243 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20244 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20245 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20246 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20247 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20248 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20249 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20250 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20251 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20252 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20253 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20254 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20255 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20256 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20257 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20258 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20259 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20260 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20261 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20262 ; GFX10-CU-NEXT: s_endpgm
20263 ;
20264 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20265 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20266 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20267 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20268 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20269 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20270 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20271 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20272 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20273 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20274 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20275 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20276 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20277 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20278 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20279 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20280 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20281 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20282 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20283 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20284 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20285 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20286 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20287 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20288 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20289 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20290 ; SKIP-CACHE-INV-NEXT: s_endpgm
20291 ;
20292 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20293 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20294 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20295 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20296 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20297 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20298 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20299 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20300 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20301 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20302 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20303 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20304 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20305 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20306 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20307 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20308 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20309 ;
20310 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20311 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20312 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20313 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20314 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20315 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20316 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20317 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20318 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20319 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20320 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20321 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20322 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20323 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20324 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20325 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20326 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20327 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20328 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20329 ;
20330 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20331 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20332 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20333 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20334 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20335 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20336 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20337 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20338 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20339 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20340 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20341 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20342 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20343 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20344 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20345 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20346 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20347 ;
20348 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20349 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20350 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20351 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20352 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20353 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20354 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20355 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20356 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20357 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20358 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20359 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20360 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20361 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20362 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20363 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20364 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20365 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20366 ; GFX940-TGSPLIT-NEXT: s_endpgm
20367 ;
20368 ; GFX11-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20369 ; GFX11-WGP: ; %bb.0: ; %entry
20370 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20371 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20372 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20373 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20374 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20375 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20376 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20377 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20378 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20379 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20380 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20381 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20382 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20383 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20384 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20385 ; GFX11-WGP-NEXT: buffer_gl0_inv
20386 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20387 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20388 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20389 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20390 ; GFX11-WGP-NEXT: s_endpgm
20391 ;
20392 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20393 ; GFX11-CU: ; %bb.0: ; %entry
20394 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20395 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20396 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20397 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20398 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20399 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20400 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20401 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20402 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20403 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20404 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20405 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20406 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20407 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20408 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20409 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20410 ; GFX11-CU-NEXT: s_endpgm
20411 ;
20412 ; GFX12-WGP-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20413 ; GFX12-WGP: ; %bb.0: ; %entry
20414 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20415 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20416 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20417 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20418 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20419 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20420 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20421 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20423 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20424 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20425 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20426 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20427 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20428 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20429 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20430 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20431 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20432 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20433 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20434 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20436 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20437 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20438 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20439 ; GFX12-WGP-NEXT: s_endpgm
20440 ;
20441 ; GFX12-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20442 ; GFX12-CU: ; %bb.0: ; %entry
20443 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20444 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20445 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20446 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20447 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20448 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20449 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20450 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20451 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20452 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20453 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20454 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
20455 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20456 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20457 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
20458 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20459 ; GFX12-CU-NEXT: s_endpgm
20460 ptr %out, i32 %in, i32 %old) {
20461 entry:
20462 %gep = getelementptr i32, ptr %out, i32 4
20463 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
20464 %val0 = extractvalue { i32, i1 } %val, 0
20465 store i32 %val0, ptr %out, align 4
20466 ret void
20467 }
20469 define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
20470 ; GFX7-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20471 ; GFX7: ; %bb.0: ; %entry
20472 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20473 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20474 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20475 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20476 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20477 ; GFX7-NEXT: s_mov_b32 s6, s4
20478 ; GFX7-NEXT: s_mov_b32 s7, s5
20479 ; GFX7-NEXT: s_mov_b32 s11, s12
20480 ; GFX7-NEXT: s_mov_b32 s10, s13
20481 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20482 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20483 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20484 ; GFX7-NEXT: s_mov_b32 s7, s10
20485 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20486 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20487 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20488 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20489 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20490 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20491 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20492 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20493 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20494 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20495 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20496 ; GFX7-NEXT: s_endpgm
20497 ;
20498 ; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20499 ; GFX10-WGP: ; %bb.0: ; %entry
20500 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20501 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20502 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20503 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20504 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20505 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20506 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20507 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20508 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20509 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20510 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20511 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20512 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20513 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20514 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20515 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20516 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20517 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20518 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20519 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20520 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20521 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20522 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20523 ; GFX10-WGP-NEXT: buffer_gl0_inv
20524 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20525 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20526 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20527 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20528 ; GFX10-WGP-NEXT: s_endpgm
20529 ;
20530 ; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20531 ; GFX10-CU: ; %bb.0: ; %entry
20532 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20533 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20534 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20535 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20536 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20537 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20538 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20539 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20540 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20541 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20542 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20543 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20544 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20545 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20546 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20547 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20548 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20549 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20550 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20551 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20552 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20553 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20554 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20555 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20556 ; GFX10-CU-NEXT: s_endpgm
20557 ;
20558 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20559 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20560 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20561 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20562 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20563 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20564 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20565 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20566 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20567 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20568 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20569 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20570 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20571 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20572 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20573 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20574 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20575 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20576 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20577 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20578 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20579 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20580 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20581 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20582 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20583 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20584 ; SKIP-CACHE-INV-NEXT: s_endpgm
20585 ;
20586 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20587 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20588 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20589 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20590 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20591 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20592 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20593 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20594 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20595 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20596 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20597 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20598 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20599 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20600 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20601 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20602 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20603 ;
20604 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20605 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20606 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20607 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20608 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20609 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20610 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20611 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20612 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20613 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20614 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20615 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20616 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20617 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20618 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20619 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20620 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20621 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20622 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20623 ;
20624 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20625 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20626 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20627 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20628 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20629 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20630 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20631 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20632 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20633 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20634 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20635 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20636 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20637 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20638 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20639 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20640 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20641 ;
20642 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20643 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20644 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20645 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20646 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20647 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20648 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20649 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20650 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20651 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20652 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20653 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20654 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20655 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20656 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20657 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20658 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20659 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20660 ; GFX940-TGSPLIT-NEXT: s_endpgm
20661 ;
20662 ; GFX11-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20663 ; GFX11-WGP: ; %bb.0: ; %entry
20664 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20665 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20666 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20667 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20668 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20669 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20670 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20671 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20672 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20673 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20674 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20675 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20676 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20677 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20678 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20679 ; GFX11-WGP-NEXT: buffer_gl0_inv
20680 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20681 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20682 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20683 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20684 ; GFX11-WGP-NEXT: s_endpgm
20685 ;
20686 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20687 ; GFX11-CU: ; %bb.0: ; %entry
20688 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20689 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20690 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20691 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20692 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20693 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20694 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20695 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20696 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20697 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20698 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20699 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20700 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20701 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20702 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20703 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20704 ; GFX11-CU-NEXT: s_endpgm
20705 ;
20706 ; GFX12-WGP-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20707 ; GFX12-WGP: ; %bb.0: ; %entry
20708 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20709 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20710 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20711 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20712 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20713 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
20714 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
20715 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20716 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
20717 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20718 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20719 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20720 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20721 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20722 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20723 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20724 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20725 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20726 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20727 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20728 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20729 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
20730 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
20731 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
20732 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
20733 ; GFX12-WGP-NEXT: s_endpgm
20734 ;
20735 ; GFX12-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20736 ; GFX12-CU: ; %bb.0: ; %entry
20737 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20738 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20739 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20740 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20741 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20742 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
20743 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
20744 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20745 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
20746 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20747 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20748 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
20749 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
20750 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
20751 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
20752 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
20753 ; GFX12-CU-NEXT: s_endpgm
20754 ptr %out, i32 %in, i32 %old) {
20755 entry:
20756 %gep = getelementptr i32, ptr %out, i32 4
20757 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
20758 %val0 = extractvalue { i32, i1 } %val, 0
20759 store i32 %val0, ptr %out, align 4
20760 ret void
20761 }
20763 define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
20764 ; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20765 ; GFX7: ; %bb.0: ; %entry
20766 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20767 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20768 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20769 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20770 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20771 ; GFX7-NEXT: s_mov_b32 s6, s4
20772 ; GFX7-NEXT: s_mov_b32 s7, s5
20773 ; GFX7-NEXT: s_mov_b32 s11, s12
20774 ; GFX7-NEXT: s_mov_b32 s10, s13
20775 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20776 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20777 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20778 ; GFX7-NEXT: s_mov_b32 s7, s10
20779 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20780 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20781 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20782 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20783 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20784 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20785 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20786 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20787 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20788 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20789 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20790 ; GFX7-NEXT: s_endpgm
20791 ;
20792 ; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20793 ; GFX10-WGP: ; %bb.0: ; %entry
20794 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20795 ; GFX10-WGP-NEXT: s_load_dword s9, s[6:7], 0x8
20796 ; GFX10-WGP-NEXT: s_load_dword s8, s[6:7], 0xc
20797 ; GFX10-WGP-NEXT: s_mov_b64 s[12:13], 16
20798 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20799 ; GFX10-WGP-NEXT: s_mov_b32 s6, s4
20800 ; GFX10-WGP-NEXT: s_mov_b32 s7, s5
20801 ; GFX10-WGP-NEXT: s_mov_b32 s11, s12
20802 ; GFX10-WGP-NEXT: s_mov_b32 s10, s13
20803 ; GFX10-WGP-NEXT: s_add_u32 s6, s6, s11
20804 ; GFX10-WGP-NEXT: s_addc_u32 s10, s7, s10
20805 ; GFX10-WGP-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20806 ; GFX10-WGP-NEXT: s_mov_b32 s7, s10
20807 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s9
20808 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s8
20809 ; GFX10-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20810 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, v0
20811 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
20812 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20813 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20814 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20815 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20816 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20817 ; GFX10-WGP-NEXT: buffer_gl0_inv
20818 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
20819 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
20820 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20821 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
20822 ; GFX10-WGP-NEXT: s_endpgm
20823 ;
20824 ; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20825 ; GFX10-CU: ; %bb.0: ; %entry
20826 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20827 ; GFX10-CU-NEXT: s_load_dword s9, s[6:7], 0x8
20828 ; GFX10-CU-NEXT: s_load_dword s8, s[6:7], 0xc
20829 ; GFX10-CU-NEXT: s_mov_b64 s[12:13], 16
20830 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20831 ; GFX10-CU-NEXT: s_mov_b32 s6, s4
20832 ; GFX10-CU-NEXT: s_mov_b32 s7, s5
20833 ; GFX10-CU-NEXT: s_mov_b32 s11, s12
20834 ; GFX10-CU-NEXT: s_mov_b32 s10, s13
20835 ; GFX10-CU-NEXT: s_add_u32 s6, s6, s11
20836 ; GFX10-CU-NEXT: s_addc_u32 s10, s7, s10
20837 ; GFX10-CU-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20838 ; GFX10-CU-NEXT: s_mov_b32 s7, s10
20839 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s9
20840 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s8
20841 ; GFX10-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20842 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, v0
20843 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
20844 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20845 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20846 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
20847 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
20848 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20849 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
20850 ; GFX10-CU-NEXT: s_endpgm
20851 ;
20852 ; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20853 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20854 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20855 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20856 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20857 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[8:9], 16
20858 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20859 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s0
20860 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s1
20861 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
20862 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
20863 ; SKIP-CACHE-INV-NEXT: s_add_u32 s2, s2, s7
20864 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s6, s3, s6
20865 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr2 killed $sgpr2 def $sgpr2_sgpr3
20866 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20867 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s5
20868 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
20869 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20870 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v0
20871 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
20872 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
20873 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20874 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
20875 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
20876 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20877 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
20878 ; SKIP-CACHE-INV-NEXT: s_endpgm
20879 ;
20880 ; GFX90A-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20881 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20882 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20883 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20884 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20885 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20886 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20887 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20888 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20889 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20890 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20891 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20892 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20893 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20894 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20895 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
20896 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20897 ;
20898 ; GFX90A-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20899 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20900 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20901 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20902 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20903 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20904 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20905 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20906 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s6
20907 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20908 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20909 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20910 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20911 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
20912 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20913 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20914 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
20915 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
20916 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20917 ;
20918 ; GFX940-NOTTGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20919 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20920 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20921 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20922 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20923 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20924 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20925 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20926 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20927 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20928 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20929 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20930 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20931 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20932 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20933 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20934 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20935 ;
20936 ; GFX940-TGSPLIT-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20937 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20938 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20939 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20940 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20941 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20942 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20943 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20944 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
20945 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20946 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v0
20947 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20948 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20949 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
20950 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20951 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20952 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
20953 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
20954 ; GFX940-TGSPLIT-NEXT: s_endpgm
20955 ;
20956 ; GFX11-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20957 ; GFX11-WGP: ; %bb.0: ; %entry
20958 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20959 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20960 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20961 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20962 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20963 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s3
20964 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
20965 ; GFX11-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20966 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, v0
20967 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20968 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20969 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20970 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20971 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20972 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20973 ; GFX11-WGP-NEXT: buffer_gl0_inv
20974 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
20975 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
20976 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20977 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
20978 ; GFX11-WGP-NEXT: s_endpgm
20979 ;
20980 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
20981 ; GFX11-CU: ; %bb.0: ; %entry
20982 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20983 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20984 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20985 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20986 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20987 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s3
20988 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
20989 ; GFX11-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20990 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, v0
20991 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20992 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20993 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
20994 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
20995 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
20996 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
20997 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
20998 ; GFX11-CU-NEXT: s_endpgm
20999 ;
21000 ; GFX12-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21001 ; GFX12-WGP: ; %bb.0: ; %entry
21002 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
21003 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21004 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21005 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21006 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21007 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s3
21008 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
21009 ; GFX12-WGP-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21010 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, v0
21011 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21012 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21013 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
21014 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21015 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21016 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21017 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21018 ; GFX12-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
21019 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21020 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21021 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21022 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
21023 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
21024 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
21025 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
21026 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
21027 ; GFX12-WGP-NEXT: s_endpgm
21028 ;
21029 ; GFX12-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21030 ; GFX12-CU: ; %bb.0: ; %entry
21031 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
21032 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21033 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21034 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21035 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21036 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s3
21037 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
21038 ; GFX12-CU-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21039 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, v0
21040 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21041 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21042 ; GFX12-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 th:TH_ATOMIC_RETURN
21043 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
21044 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
21045 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
21046 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
21047 ; GFX12-CU-NEXT: s_endpgm
21048 ptr %out, i32 %in, i32 %old) {
21049 entry:
21050 %gep = getelementptr i32, ptr %out, i32 4
21051 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
21052 %val0 = extractvalue { i32, i1 } %val, 0
21053 store i32 %val0, ptr %out, align 4
21054 ret void
21055 }