| blob | 350ab8d48da81be48fc775250e800040321ded83 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
11 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
13 define amdgpu_kernel void @flat_wavefront_unordered_load(
14 ; GFX7-LABEL: flat_wavefront_unordered_load:
15 ; GFX7: ; %bb.0: ; %entry
16 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
17 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
19 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
20 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
21 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
22 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
23 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
24 ; GFX7-NEXT: flat_store_dword v[0:1], v2
25 ; GFX7-NEXT: s_endpgm
26 ;
27 ; GFX10-WGP-LABEL: flat_wavefront_unordered_load:
28 ; GFX10-WGP: ; %bb.0: ; %entry
29 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
30 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
31 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
32 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
33 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
34 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
35 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
36 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
37 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
38 ; GFX10-WGP-NEXT: s_endpgm
39 ;
40 ; GFX10-CU-LABEL: flat_wavefront_unordered_load:
41 ; GFX10-CU: ; %bb.0: ; %entry
42 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
43 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
44 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
45 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
46 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
47 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
48 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
49 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
50 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
51 ; GFX10-CU-NEXT: s_endpgm
52 ;
53 ; SKIP-CACHE-INV-LABEL: flat_wavefront_unordered_load:
54 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
55 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
56 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
57 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
58 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
59 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
60 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
61 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
62 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
63 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
64 ; SKIP-CACHE-INV-NEXT: s_endpgm
65 ;
66 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_unordered_load:
67 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
68 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
69 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
70 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
71 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
72 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
73 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
74 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
75 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
76 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
77 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
78 ;
79 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_unordered_load:
80 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
81 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
82 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
83 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
84 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
85 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
86 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
87 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
88 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
89 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
90 ; GFX90A-TGSPLIT-NEXT: s_endpgm
91 ;
92 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_unordered_load:
93 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
94 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
95 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
96 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
97 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
98 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
99 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
100 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
101 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
102 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
103 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
104 ;
105 ; GFX940-TGSPLIT-LABEL: flat_wavefront_unordered_load:
106 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
107 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
108 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
109 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
110 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
111 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
112 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
113 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
114 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
115 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
116 ; GFX940-TGSPLIT-NEXT: s_endpgm
117 ;
118 ; GFX11-WGP-LABEL: flat_wavefront_unordered_load:
119 ; GFX11-WGP: ; %bb.0: ; %entry
120 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
121 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
122 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
123 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
124 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
125 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
126 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
127 ; GFX11-WGP-NEXT: s_endpgm
128 ;
129 ; GFX11-CU-LABEL: flat_wavefront_unordered_load:
130 ; GFX11-CU: ; %bb.0: ; %entry
131 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
132 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
133 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
134 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
135 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
136 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
137 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
138 ; GFX11-CU-NEXT: s_endpgm
139 ptr %in, ptr %out) {
140 entry:
141 %val = load atomic i32, ptr %in syncscope("wavefront") unordered, align 4
142 store i32 %val, ptr %out
143 ret void
144 }
146 define amdgpu_kernel void @flat_wavefront_monotonic_load(
147 ; GFX7-LABEL: flat_wavefront_monotonic_load:
148 ; GFX7: ; %bb.0: ; %entry
149 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
150 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
151 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
152 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
153 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
154 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
155 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
156 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
157 ; GFX7-NEXT: flat_store_dword v[0:1], v2
158 ; GFX7-NEXT: s_endpgm
159 ;
160 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_load:
161 ; GFX10-WGP: ; %bb.0: ; %entry
162 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
163 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
164 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
165 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
166 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
167 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
168 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
169 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
170 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
171 ; GFX10-WGP-NEXT: s_endpgm
172 ;
173 ; GFX10-CU-LABEL: flat_wavefront_monotonic_load:
174 ; GFX10-CU: ; %bb.0: ; %entry
175 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
176 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
177 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
178 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
179 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
180 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
181 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
182 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
183 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
184 ; GFX10-CU-NEXT: s_endpgm
185 ;
186 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_load:
187 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
188 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
189 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
190 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
191 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
192 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
193 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
194 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
195 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
196 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
197 ; SKIP-CACHE-INV-NEXT: s_endpgm
198 ;
199 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_load:
200 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
201 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
202 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
203 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
204 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
205 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
206 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
207 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
208 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
209 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
210 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
211 ;
212 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_load:
213 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
214 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
215 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
216 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
217 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
218 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
219 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
220 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
221 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
222 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
223 ; GFX90A-TGSPLIT-NEXT: s_endpgm
224 ;
225 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_load:
226 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
227 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
228 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
229 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
230 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
231 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
232 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
233 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
234 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
235 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
236 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
237 ;
238 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_load:
239 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
240 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
241 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
242 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
243 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
244 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
245 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
246 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
247 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
248 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
249 ; GFX940-TGSPLIT-NEXT: s_endpgm
250 ;
251 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_load:
252 ; GFX11-WGP: ; %bb.0: ; %entry
253 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
254 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
255 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
256 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
257 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
258 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
259 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
260 ; GFX11-WGP-NEXT: s_endpgm
261 ;
262 ; GFX11-CU-LABEL: flat_wavefront_monotonic_load:
263 ; GFX11-CU: ; %bb.0: ; %entry
264 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
265 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
266 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
267 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
268 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
269 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
270 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
271 ; GFX11-CU-NEXT: s_endpgm
272 ptr %in, ptr %out) {
273 entry:
274 %val = load atomic i32, ptr %in syncscope("wavefront") monotonic, align 4
275 store i32 %val, ptr %out
276 ret void
277 }
279 define amdgpu_kernel void @flat_wavefront_acquire_load(
280 ; GFX7-LABEL: flat_wavefront_acquire_load:
281 ; GFX7: ; %bb.0: ; %entry
282 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
283 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
284 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
285 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
286 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
287 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
288 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
289 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
290 ; GFX7-NEXT: flat_store_dword v[0:1], v2
291 ; GFX7-NEXT: s_endpgm
292 ;
293 ; GFX10-WGP-LABEL: flat_wavefront_acquire_load:
294 ; GFX10-WGP: ; %bb.0: ; %entry
295 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
296 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
297 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
298 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
299 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
300 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
301 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
302 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
303 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
304 ; GFX10-WGP-NEXT: s_endpgm
305 ;
306 ; GFX10-CU-LABEL: flat_wavefront_acquire_load:
307 ; GFX10-CU: ; %bb.0: ; %entry
308 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
309 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
310 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
311 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
312 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
313 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
314 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
315 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
316 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
317 ; GFX10-CU-NEXT: s_endpgm
318 ;
319 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_load:
320 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
321 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
322 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
323 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
324 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
325 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
326 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
327 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
328 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
329 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
330 ; SKIP-CACHE-INV-NEXT: s_endpgm
331 ;
332 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_load:
333 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
334 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
335 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
336 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
337 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
338 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
339 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
340 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
341 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
342 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
343 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
344 ;
345 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_load:
346 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
347 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
348 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
349 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
350 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
351 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
352 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
353 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
354 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
355 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
356 ; GFX90A-TGSPLIT-NEXT: s_endpgm
357 ;
358 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_load:
359 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
360 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
361 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
362 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
363 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
364 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
365 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
366 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
368 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
369 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
370 ;
371 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_load:
372 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
373 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
374 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
375 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
376 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
377 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
378 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
379 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
380 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
381 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
382 ; GFX940-TGSPLIT-NEXT: s_endpgm
383 ;
384 ; GFX11-WGP-LABEL: flat_wavefront_acquire_load:
385 ; GFX11-WGP: ; %bb.0: ; %entry
386 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
387 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
388 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
389 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
390 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
391 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
392 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
393 ; GFX11-WGP-NEXT: s_endpgm
394 ;
395 ; GFX11-CU-LABEL: flat_wavefront_acquire_load:
396 ; GFX11-CU: ; %bb.0: ; %entry
397 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
398 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
399 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
400 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
401 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
402 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
403 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
404 ; GFX11-CU-NEXT: s_endpgm
405 ptr %in, ptr %out) {
406 entry:
407 %val = load atomic i32, ptr %in syncscope("wavefront") acquire, align 4
408 store i32 %val, ptr %out
409 ret void
410 }
412 define amdgpu_kernel void @flat_wavefront_seq_cst_load(
413 ; GFX7-LABEL: flat_wavefront_seq_cst_load:
414 ; GFX7: ; %bb.0: ; %entry
415 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
416 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
417 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
418 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
419 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
420 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
421 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
422 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
423 ; GFX7-NEXT: flat_store_dword v[0:1], v2
424 ; GFX7-NEXT: s_endpgm
425 ;
426 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_load:
427 ; GFX10-WGP: ; %bb.0: ; %entry
428 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
429 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
430 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
431 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
432 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
433 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
434 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
435 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
436 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
437 ; GFX10-WGP-NEXT: s_endpgm
438 ;
439 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_load:
440 ; GFX10-CU: ; %bb.0: ; %entry
441 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
442 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
443 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
444 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
445 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
446 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
447 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
448 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
449 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
450 ; GFX10-CU-NEXT: s_endpgm
451 ;
452 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_load:
453 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
454 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
455 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
456 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
457 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
458 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
459 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
460 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
461 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
462 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
463 ; SKIP-CACHE-INV-NEXT: s_endpgm
464 ;
465 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_load:
466 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
467 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
468 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
469 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
470 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
471 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
472 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
473 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
474 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
475 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
476 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
477 ;
478 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_load:
479 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
480 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
481 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
482 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
483 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
484 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
485 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
486 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
487 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
488 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
489 ; GFX90A-TGSPLIT-NEXT: s_endpgm
490 ;
491 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_load:
492 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
493 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
494 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
495 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
496 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
497 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
498 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
499 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
500 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
501 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
502 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
503 ;
504 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_load:
505 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
506 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
507 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
508 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
509 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
510 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
511 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
512 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
513 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
514 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
515 ; GFX940-TGSPLIT-NEXT: s_endpgm
516 ;
517 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_load:
518 ; GFX11-WGP: ; %bb.0: ; %entry
519 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
520 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
521 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
522 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
523 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
524 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
525 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
526 ; GFX11-WGP-NEXT: s_endpgm
527 ;
528 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_load:
529 ; GFX11-CU: ; %bb.0: ; %entry
530 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
531 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
532 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
533 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
534 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
535 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
536 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
537 ; GFX11-CU-NEXT: s_endpgm
538 ptr %in, ptr %out) {
539 entry:
540 %val = load atomic i32, ptr %in syncscope("wavefront") seq_cst, align 4
541 store i32 %val, ptr %out
542 ret void
543 }
545 define amdgpu_kernel void @flat_wavefront_unordered_store(
546 ; GFX7-LABEL: flat_wavefront_unordered_store:
547 ; GFX7: ; %bb.0: ; %entry
548 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
549 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
550 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
551 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
552 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
553 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
554 ; GFX7-NEXT: flat_store_dword v[0:1], v2
555 ; GFX7-NEXT: s_endpgm
556 ;
557 ; GFX10-WGP-LABEL: flat_wavefront_unordered_store:
558 ; GFX10-WGP: ; %bb.0: ; %entry
559 ; GFX10-WGP-NEXT: s_clause 0x1
560 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
561 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
562 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
563 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
564 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
565 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
566 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
567 ; GFX10-WGP-NEXT: s_endpgm
568 ;
569 ; GFX10-CU-LABEL: flat_wavefront_unordered_store:
570 ; GFX10-CU: ; %bb.0: ; %entry
571 ; GFX10-CU-NEXT: s_clause 0x1
572 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
573 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
574 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
575 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
576 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
577 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
578 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
579 ; GFX10-CU-NEXT: s_endpgm
580 ;
581 ; SKIP-CACHE-INV-LABEL: flat_wavefront_unordered_store:
582 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
583 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
584 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
585 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
586 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
587 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
588 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
589 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
590 ; SKIP-CACHE-INV-NEXT: s_endpgm
591 ;
592 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_unordered_store:
593 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
594 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
595 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
596 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
597 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
598 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
599 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
600 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
601 ;
602 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_unordered_store:
603 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
604 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
605 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
606 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
607 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
608 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
609 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
610 ; GFX90A-TGSPLIT-NEXT: s_endpgm
611 ;
612 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_unordered_store:
613 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
614 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
615 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
616 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
617 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
618 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
619 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
620 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
621 ;
622 ; GFX940-TGSPLIT-LABEL: flat_wavefront_unordered_store:
623 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
624 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
625 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
626 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
627 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
628 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
629 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
630 ; GFX940-TGSPLIT-NEXT: s_endpgm
631 ;
632 ; GFX11-WGP-LABEL: flat_wavefront_unordered_store:
633 ; GFX11-WGP: ; %bb.0: ; %entry
634 ; GFX11-WGP-NEXT: s_clause 0x1
635 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
636 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
637 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
638 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
639 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
640 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
641 ; GFX11-WGP-NEXT: s_endpgm
642 ;
643 ; GFX11-CU-LABEL: flat_wavefront_unordered_store:
644 ; GFX11-CU: ; %bb.0: ; %entry
645 ; GFX11-CU-NEXT: s_clause 0x1
646 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
647 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
648 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
649 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
650 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
651 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
652 ; GFX11-CU-NEXT: s_endpgm
653 i32 %in, ptr %out) {
654 entry:
655 store atomic i32 %in, ptr %out syncscope("wavefront") unordered, align 4
656 ret void
657 }
659 define amdgpu_kernel void @flat_wavefront_monotonic_store(
660 ; GFX7-LABEL: flat_wavefront_monotonic_store:
661 ; GFX7: ; %bb.0: ; %entry
662 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
663 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
664 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
665 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
666 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
667 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
668 ; GFX7-NEXT: flat_store_dword v[0:1], v2
669 ; GFX7-NEXT: s_endpgm
670 ;
671 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_store:
672 ; GFX10-WGP: ; %bb.0: ; %entry
673 ; GFX10-WGP-NEXT: s_clause 0x1
674 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
675 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
676 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
677 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
678 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
679 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
680 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
681 ; GFX10-WGP-NEXT: s_endpgm
682 ;
683 ; GFX10-CU-LABEL: flat_wavefront_monotonic_store:
684 ; GFX10-CU: ; %bb.0: ; %entry
685 ; GFX10-CU-NEXT: s_clause 0x1
686 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
687 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
688 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
689 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
690 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
691 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
692 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
693 ; GFX10-CU-NEXT: s_endpgm
694 ;
695 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_store:
696 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
697 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
698 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
699 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
700 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
701 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
702 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
703 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
704 ; SKIP-CACHE-INV-NEXT: s_endpgm
705 ;
706 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_store:
707 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
708 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
709 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
710 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
711 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
712 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
713 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
714 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
715 ;
716 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_store:
717 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
718 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
719 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
720 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
721 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
722 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
723 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
724 ; GFX90A-TGSPLIT-NEXT: s_endpgm
725 ;
726 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_store:
727 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
728 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
729 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
730 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
731 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
732 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
733 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
734 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
735 ;
736 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_store:
737 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
738 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
739 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
740 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
741 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
742 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
743 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
744 ; GFX940-TGSPLIT-NEXT: s_endpgm
745 ;
746 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_store:
747 ; GFX11-WGP: ; %bb.0: ; %entry
748 ; GFX11-WGP-NEXT: s_clause 0x1
749 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
750 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
751 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
752 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
753 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
754 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
755 ; GFX11-WGP-NEXT: s_endpgm
756 ;
757 ; GFX11-CU-LABEL: flat_wavefront_monotonic_store:
758 ; GFX11-CU: ; %bb.0: ; %entry
759 ; GFX11-CU-NEXT: s_clause 0x1
760 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
761 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
762 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
763 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
764 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
765 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
766 ; GFX11-CU-NEXT: s_endpgm
767 i32 %in, ptr %out) {
768 entry:
769 store atomic i32 %in, ptr %out syncscope("wavefront") monotonic, align 4
770 ret void
771 }
773 define amdgpu_kernel void @flat_wavefront_release_store(
774 ; GFX7-LABEL: flat_wavefront_release_store:
775 ; GFX7: ; %bb.0: ; %entry
776 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
777 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
778 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
779 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
780 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
781 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
782 ; GFX7-NEXT: flat_store_dword v[0:1], v2
783 ; GFX7-NEXT: s_endpgm
784 ;
785 ; GFX10-WGP-LABEL: flat_wavefront_release_store:
786 ; GFX10-WGP: ; %bb.0: ; %entry
787 ; GFX10-WGP-NEXT: s_clause 0x1
788 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
789 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
790 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
791 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
792 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
793 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
794 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
795 ; GFX10-WGP-NEXT: s_endpgm
796 ;
797 ; GFX10-CU-LABEL: flat_wavefront_release_store:
798 ; GFX10-CU: ; %bb.0: ; %entry
799 ; GFX10-CU-NEXT: s_clause 0x1
800 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
801 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
802 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
803 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
804 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
805 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
806 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
807 ; GFX10-CU-NEXT: s_endpgm
808 ;
809 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_store:
810 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
811 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
812 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
813 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
814 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
815 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
816 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
817 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
818 ; SKIP-CACHE-INV-NEXT: s_endpgm
819 ;
820 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_store:
821 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
822 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
823 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
824 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
825 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
826 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
827 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
828 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
829 ;
830 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_store:
831 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
832 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
833 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
834 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
835 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
836 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
837 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
838 ; GFX90A-TGSPLIT-NEXT: s_endpgm
839 ;
840 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_store:
841 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
842 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
843 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
844 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
845 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
846 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
847 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
848 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
849 ;
850 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_store:
851 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
852 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
853 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
854 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
855 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
856 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
857 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
858 ; GFX940-TGSPLIT-NEXT: s_endpgm
859 ;
860 ; GFX11-WGP-LABEL: flat_wavefront_release_store:
861 ; GFX11-WGP: ; %bb.0: ; %entry
862 ; GFX11-WGP-NEXT: s_clause 0x1
863 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
864 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
865 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
866 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
867 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
868 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
869 ; GFX11-WGP-NEXT: s_endpgm
870 ;
871 ; GFX11-CU-LABEL: flat_wavefront_release_store:
872 ; GFX11-CU: ; %bb.0: ; %entry
873 ; GFX11-CU-NEXT: s_clause 0x1
874 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
875 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
876 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
877 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
878 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
879 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
880 ; GFX11-CU-NEXT: s_endpgm
881 i32 %in, ptr %out) {
882 entry:
883 store atomic i32 %in, ptr %out syncscope("wavefront") release, align 4
884 ret void
885 }
887 define amdgpu_kernel void @flat_wavefront_seq_cst_store(
888 ; GFX7-LABEL: flat_wavefront_seq_cst_store:
889 ; GFX7: ; %bb.0: ; %entry
890 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
891 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
892 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
893 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
894 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
895 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
896 ; GFX7-NEXT: flat_store_dword v[0:1], v2
897 ; GFX7-NEXT: s_endpgm
898 ;
899 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_store:
900 ; GFX10-WGP: ; %bb.0: ; %entry
901 ; GFX10-WGP-NEXT: s_clause 0x1
902 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
903 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
904 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
905 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
906 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
907 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
908 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
909 ; GFX10-WGP-NEXT: s_endpgm
910 ;
911 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_store:
912 ; GFX10-CU: ; %bb.0: ; %entry
913 ; GFX10-CU-NEXT: s_clause 0x1
914 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
915 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
916 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
917 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
918 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
919 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
920 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
921 ; GFX10-CU-NEXT: s_endpgm
922 ;
923 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_store:
924 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
925 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
926 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
927 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
928 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
929 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
930 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
931 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
932 ; SKIP-CACHE-INV-NEXT: s_endpgm
933 ;
934 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_store:
935 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
936 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
937 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
938 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
939 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
940 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
941 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
942 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
943 ;
944 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_store:
945 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
946 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
947 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
948 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
949 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
950 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
951 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
952 ; GFX90A-TGSPLIT-NEXT: s_endpgm
953 ;
954 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_store:
955 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
956 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
957 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
958 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
959 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
960 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
961 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
962 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
963 ;
964 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_store:
965 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
966 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
967 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
968 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
969 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
970 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
971 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
972 ; GFX940-TGSPLIT-NEXT: s_endpgm
973 ;
974 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_store:
975 ; GFX11-WGP: ; %bb.0: ; %entry
976 ; GFX11-WGP-NEXT: s_clause 0x1
977 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
978 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
979 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
980 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
981 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
982 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
983 ; GFX11-WGP-NEXT: s_endpgm
984 ;
985 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_store:
986 ; GFX11-CU: ; %bb.0: ; %entry
987 ; GFX11-CU-NEXT: s_clause 0x1
988 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
989 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
990 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
991 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
992 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
993 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
994 ; GFX11-CU-NEXT: s_endpgm
995 i32 %in, ptr %out) {
996 entry:
997 store atomic i32 %in, ptr %out syncscope("wavefront") seq_cst, align 4
998 ret void
999 }
1001 define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
1002 ; GFX7-LABEL: flat_wavefront_monotonic_atomicrmw:
1003 ; GFX7: ; %bb.0: ; %entry
1004 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1005 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1006 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1007 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1008 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1009 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1010 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1011 ; GFX7-NEXT: s_endpgm
1012 ;
1013 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_atomicrmw:
1014 ; GFX10-WGP: ; %bb.0: ; %entry
1015 ; GFX10-WGP-NEXT: s_clause 0x1
1016 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1017 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1018 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1019 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1020 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1021 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1022 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1023 ; GFX10-WGP-NEXT: s_endpgm
1024 ;
1025 ; GFX10-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
1026 ; GFX10-CU: ; %bb.0: ; %entry
1027 ; GFX10-CU-NEXT: s_clause 0x1
1028 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1029 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1030 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1031 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1032 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1033 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1034 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1035 ; GFX10-CU-NEXT: s_endpgm
1036 ;
1037 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_atomicrmw:
1038 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1039 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1040 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1041 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1042 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1043 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1044 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1045 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1046 ; SKIP-CACHE-INV-NEXT: s_endpgm
1047 ;
1048 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
1049 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1050 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1051 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1052 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1053 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1054 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1055 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1056 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1057 ;
1058 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
1059 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1060 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1061 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1062 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1063 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1064 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1065 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1066 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1067 ;
1068 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
1069 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1070 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1071 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1072 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1073 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1074 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1075 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1076 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1077 ;
1078 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_atomicrmw:
1079 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1080 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1081 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1082 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1083 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1084 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1085 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1086 ; GFX940-TGSPLIT-NEXT: s_endpgm
1087 ;
1088 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_atomicrmw:
1089 ; GFX11-WGP: ; %bb.0: ; %entry
1090 ; GFX11-WGP-NEXT: s_clause 0x1
1091 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1092 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1093 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1094 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1095 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1096 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1097 ; GFX11-WGP-NEXT: s_endpgm
1098 ;
1099 ; GFX11-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
1100 ; GFX11-CU: ; %bb.0: ; %entry
1101 ; GFX11-CU-NEXT: s_clause 0x1
1102 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1103 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1104 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1105 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1106 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1107 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1108 ; GFX11-CU-NEXT: s_endpgm
1109 ptr %out, i32 %in) {
1110 entry:
1111 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") monotonic
1112 ret void
1113 }
1115 define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
1116 ; GFX7-LABEL: flat_wavefront_acquire_atomicrmw:
1117 ; GFX7: ; %bb.0: ; %entry
1118 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1119 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1120 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1121 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1122 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1123 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1124 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1125 ; GFX7-NEXT: s_endpgm
1126 ;
1127 ; GFX10-WGP-LABEL: flat_wavefront_acquire_atomicrmw:
1128 ; GFX10-WGP: ; %bb.0: ; %entry
1129 ; GFX10-WGP-NEXT: s_clause 0x1
1130 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1131 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1132 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1133 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1134 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1135 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1136 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1137 ; GFX10-WGP-NEXT: s_endpgm
1138 ;
1139 ; GFX10-CU-LABEL: flat_wavefront_acquire_atomicrmw:
1140 ; GFX10-CU: ; %bb.0: ; %entry
1141 ; GFX10-CU-NEXT: s_clause 0x1
1142 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1143 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1144 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1145 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1146 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1147 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1148 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1149 ; GFX10-CU-NEXT: s_endpgm
1150 ;
1151 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_atomicrmw:
1152 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1153 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1154 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1155 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1156 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1157 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1158 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1159 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1160 ; SKIP-CACHE-INV-NEXT: s_endpgm
1161 ;
1162 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
1163 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1164 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1165 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1166 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1167 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1168 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1169 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1170 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1171 ;
1172 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
1173 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1174 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1175 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1176 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1177 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1178 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1179 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1180 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1181 ;
1182 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
1183 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1184 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1185 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1186 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1187 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1188 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1189 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1190 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1191 ;
1192 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_atomicrmw:
1193 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1194 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1195 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1196 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1197 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1198 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1199 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1200 ; GFX940-TGSPLIT-NEXT: s_endpgm
1201 ;
1202 ; GFX11-WGP-LABEL: flat_wavefront_acquire_atomicrmw:
1203 ; GFX11-WGP: ; %bb.0: ; %entry
1204 ; GFX11-WGP-NEXT: s_clause 0x1
1205 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1206 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1207 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1208 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1209 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1210 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1211 ; GFX11-WGP-NEXT: s_endpgm
1212 ;
1213 ; GFX11-CU-LABEL: flat_wavefront_acquire_atomicrmw:
1214 ; GFX11-CU: ; %bb.0: ; %entry
1215 ; GFX11-CU-NEXT: s_clause 0x1
1216 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1217 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1218 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1219 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1220 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1221 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1222 ; GFX11-CU-NEXT: s_endpgm
1223 ptr %out, i32 %in) {
1224 entry:
1225 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
1226 ret void
1227 }
1229 define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
1230 ; GFX7-LABEL: flat_wavefront_release_atomicrmw:
1231 ; GFX7: ; %bb.0: ; %entry
1232 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1233 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1234 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1235 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1236 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1237 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1238 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1239 ; GFX7-NEXT: s_endpgm
1240 ;
1241 ; GFX10-WGP-LABEL: flat_wavefront_release_atomicrmw:
1242 ; GFX10-WGP: ; %bb.0: ; %entry
1243 ; GFX10-WGP-NEXT: s_clause 0x1
1244 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1245 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1246 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1247 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1248 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1249 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1250 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1251 ; GFX10-WGP-NEXT: s_endpgm
1252 ;
1253 ; GFX10-CU-LABEL: flat_wavefront_release_atomicrmw:
1254 ; GFX10-CU: ; %bb.0: ; %entry
1255 ; GFX10-CU-NEXT: s_clause 0x1
1256 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1257 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1258 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1259 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1260 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1261 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1262 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1263 ; GFX10-CU-NEXT: s_endpgm
1264 ;
1265 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_atomicrmw:
1266 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1267 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1268 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1269 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1270 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1271 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1272 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1273 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1274 ; SKIP-CACHE-INV-NEXT: s_endpgm
1275 ;
1276 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
1277 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1278 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1279 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1280 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1281 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1282 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1283 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1284 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1285 ;
1286 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
1287 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1288 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1289 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1290 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1291 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1292 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1293 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1294 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1295 ;
1296 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
1297 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1298 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1299 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1300 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1301 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1302 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1303 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1304 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1305 ;
1306 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_atomicrmw:
1307 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1308 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1309 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1310 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1311 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1312 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1313 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1314 ; GFX940-TGSPLIT-NEXT: s_endpgm
1315 ;
1316 ; GFX11-WGP-LABEL: flat_wavefront_release_atomicrmw:
1317 ; GFX11-WGP: ; %bb.0: ; %entry
1318 ; GFX11-WGP-NEXT: s_clause 0x1
1319 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1320 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1321 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1322 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1323 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1324 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1325 ; GFX11-WGP-NEXT: s_endpgm
1326 ;
1327 ; GFX11-CU-LABEL: flat_wavefront_release_atomicrmw:
1328 ; GFX11-CU: ; %bb.0: ; %entry
1329 ; GFX11-CU-NEXT: s_clause 0x1
1330 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1331 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1332 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1333 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1334 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1335 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1336 ; GFX11-CU-NEXT: s_endpgm
1337 ptr %out, i32 %in) {
1338 entry:
1339 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") release
1340 ret void
1341 }
1343 define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
1344 ; GFX7-LABEL: flat_wavefront_acq_rel_atomicrmw:
1345 ; GFX7: ; %bb.0: ; %entry
1346 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1347 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1348 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1349 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1350 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1351 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1352 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1353 ; GFX7-NEXT: s_endpgm
1354 ;
1355 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_atomicrmw:
1356 ; GFX10-WGP: ; %bb.0: ; %entry
1357 ; GFX10-WGP-NEXT: s_clause 0x1
1358 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1359 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1360 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1361 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1362 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1363 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1364 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1365 ; GFX10-WGP-NEXT: s_endpgm
1366 ;
1367 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
1368 ; GFX10-CU: ; %bb.0: ; %entry
1369 ; GFX10-CU-NEXT: s_clause 0x1
1370 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1371 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1372 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1373 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1374 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1375 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1376 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1377 ; GFX10-CU-NEXT: s_endpgm
1378 ;
1379 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_atomicrmw:
1380 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1381 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1382 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1383 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1384 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1385 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1386 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1387 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1388 ; SKIP-CACHE-INV-NEXT: s_endpgm
1389 ;
1390 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
1391 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1392 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1393 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1394 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1395 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1396 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1397 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1398 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1399 ;
1400 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
1401 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1402 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1403 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1404 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1405 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1406 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1407 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1408 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1409 ;
1410 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
1411 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1412 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1413 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1414 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1415 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1416 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1417 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1418 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1419 ;
1420 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_atomicrmw:
1421 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1422 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1423 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1424 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1425 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1426 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1427 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1428 ; GFX940-TGSPLIT-NEXT: s_endpgm
1429 ;
1430 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_atomicrmw:
1431 ; GFX11-WGP: ; %bb.0: ; %entry
1432 ; GFX11-WGP-NEXT: s_clause 0x1
1433 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1434 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1435 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1436 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1437 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1438 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1439 ; GFX11-WGP-NEXT: s_endpgm
1440 ;
1441 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
1442 ; GFX11-CU: ; %bb.0: ; %entry
1443 ; GFX11-CU-NEXT: s_clause 0x1
1444 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1445 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1446 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1447 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1448 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1449 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1450 ; GFX11-CU-NEXT: s_endpgm
1451 ptr %out, i32 %in) {
1452 entry:
1453 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
1454 ret void
1455 }
1457 define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
1458 ; GFX7-LABEL: flat_wavefront_seq_cst_atomicrmw:
1459 ; GFX7: ; %bb.0: ; %entry
1460 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1461 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1462 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1463 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1464 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1465 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1466 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1467 ; GFX7-NEXT: s_endpgm
1468 ;
1469 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_atomicrmw:
1470 ; GFX10-WGP: ; %bb.0: ; %entry
1471 ; GFX10-WGP-NEXT: s_clause 0x1
1472 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1473 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1474 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1475 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1476 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1477 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1478 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
1479 ; GFX10-WGP-NEXT: s_endpgm
1480 ;
1481 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
1482 ; GFX10-CU: ; %bb.0: ; %entry
1483 ; GFX10-CU-NEXT: s_clause 0x1
1484 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1485 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1486 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1487 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1488 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1489 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1490 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
1491 ; GFX10-CU-NEXT: s_endpgm
1492 ;
1493 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_atomicrmw:
1494 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1495 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1496 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1497 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1498 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1499 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1500 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1501 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
1502 ; SKIP-CACHE-INV-NEXT: s_endpgm
1503 ;
1504 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
1505 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1506 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1507 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1508 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1509 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1510 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1511 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1512 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1513 ;
1514 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
1515 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1516 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1517 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1518 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1519 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1520 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1521 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1522 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1523 ;
1524 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
1525 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1526 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1527 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1528 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1529 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1530 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1531 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1532 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1533 ;
1534 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_atomicrmw:
1535 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1536 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1537 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1538 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1539 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1540 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1541 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
1542 ; GFX940-TGSPLIT-NEXT: s_endpgm
1543 ;
1544 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_atomicrmw:
1545 ; GFX11-WGP: ; %bb.0: ; %entry
1546 ; GFX11-WGP-NEXT: s_clause 0x1
1547 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1548 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1549 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1550 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1551 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1552 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
1553 ; GFX11-WGP-NEXT: s_endpgm
1554 ;
1555 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
1556 ; GFX11-CU: ; %bb.0: ; %entry
1557 ; GFX11-CU-NEXT: s_clause 0x1
1558 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1559 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1560 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1561 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1562 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1563 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
1564 ; GFX11-CU-NEXT: s_endpgm
1565 ptr %out, i32 %in) {
1566 entry:
1567 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
1568 ret void
1569 }
1571 define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
1572 ; GFX7-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1573 ; GFX7: ; %bb.0: ; %entry
1574 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1575 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1576 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1577 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1578 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1579 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1580 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1581 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1582 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1583 ; GFX7-NEXT: s_endpgm
1584 ;
1585 ; GFX10-WGP-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1586 ; GFX10-WGP: ; %bb.0: ; %entry
1587 ; GFX10-WGP-NEXT: s_clause 0x1
1588 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1589 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1590 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1591 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1592 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1593 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1594 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1595 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1596 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1597 ; GFX10-WGP-NEXT: s_endpgm
1598 ;
1599 ; GFX10-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1600 ; GFX10-CU: ; %bb.0: ; %entry
1601 ; GFX10-CU-NEXT: s_clause 0x1
1602 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1603 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1604 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1605 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1606 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1607 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1608 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1609 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1610 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1611 ; GFX10-CU-NEXT: s_endpgm
1612 ;
1613 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1614 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1615 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1616 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1617 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1618 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1619 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1620 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1621 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1622 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1623 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1624 ; SKIP-CACHE-INV-NEXT: s_endpgm
1625 ;
1626 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1627 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1628 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1629 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1630 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1631 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1632 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1633 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1634 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1635 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1636 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1637 ;
1638 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1639 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1640 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1641 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1642 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1643 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1644 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1645 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1646 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1647 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1648 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1649 ;
1650 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1651 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1652 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1653 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1654 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1655 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1656 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1657 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1658 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1659 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1660 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1661 ;
1662 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1663 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1664 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1665 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1666 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1667 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1668 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1669 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1670 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1671 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1672 ; GFX940-TGSPLIT-NEXT: s_endpgm
1673 ;
1674 ; GFX11-WGP-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1675 ; GFX11-WGP: ; %bb.0: ; %entry
1676 ; GFX11-WGP-NEXT: s_clause 0x1
1677 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1678 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1679 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1680 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1681 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1682 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1683 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1684 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1685 ; GFX11-WGP-NEXT: s_endpgm
1686 ;
1687 ; GFX11-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
1688 ; GFX11-CU: ; %bb.0: ; %entry
1689 ; GFX11-CU-NEXT: s_clause 0x1
1690 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1691 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1692 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1693 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1694 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1695 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1696 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1697 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1698 ; GFX11-CU-NEXT: s_endpgm
1699 ptr %out, i32 %in) {
1700 entry:
1701 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
1702 store i32 %val, ptr %out, align 4
1703 ret void
1704 }
1706 define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
1707 ; GFX7-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1708 ; GFX7: ; %bb.0: ; %entry
1709 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1710 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1711 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1712 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1713 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1714 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1715 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1716 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1717 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1718 ; GFX7-NEXT: s_endpgm
1719 ;
1720 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1721 ; GFX10-WGP: ; %bb.0: ; %entry
1722 ; GFX10-WGP-NEXT: s_clause 0x1
1723 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1724 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1725 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1726 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1727 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1728 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1729 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1730 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1731 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1732 ; GFX10-WGP-NEXT: s_endpgm
1733 ;
1734 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1735 ; GFX10-CU: ; %bb.0: ; %entry
1736 ; GFX10-CU-NEXT: s_clause 0x1
1737 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1738 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1739 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1740 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1741 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1742 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1743 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1744 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1745 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1746 ; GFX10-CU-NEXT: s_endpgm
1747 ;
1748 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1749 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1750 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1751 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1752 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1753 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1754 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1755 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1756 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1757 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1758 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1759 ; SKIP-CACHE-INV-NEXT: s_endpgm
1760 ;
1761 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1762 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1763 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1764 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1765 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1766 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1767 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1768 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1769 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1770 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1771 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1772 ;
1773 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1774 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1775 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1776 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1777 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1778 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1779 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1780 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1781 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1782 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1783 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1784 ;
1785 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1786 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1787 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1788 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1789 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1790 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1791 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1792 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1793 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1794 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1795 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1796 ;
1797 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1798 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1799 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1800 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1801 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1802 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1803 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1804 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1805 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1806 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1807 ; GFX940-TGSPLIT-NEXT: s_endpgm
1808 ;
1809 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1810 ; GFX11-WGP: ; %bb.0: ; %entry
1811 ; GFX11-WGP-NEXT: s_clause 0x1
1812 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1813 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1814 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1815 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1816 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1817 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1818 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1819 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1820 ; GFX11-WGP-NEXT: s_endpgm
1821 ;
1822 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
1823 ; GFX11-CU: ; %bb.0: ; %entry
1824 ; GFX11-CU-NEXT: s_clause 0x1
1825 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1826 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1827 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1828 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1829 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1830 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1831 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1832 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1833 ; GFX11-CU-NEXT: s_endpgm
1834 ptr %out, i32 %in) {
1835 entry:
1836 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
1837 store i32 %val, ptr %out, align 4
1838 ret void
1839 }
1841 define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
1842 ; GFX7-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1843 ; GFX7: ; %bb.0: ; %entry
1844 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1845 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
1846 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1847 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1848 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1849 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1850 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1851 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1852 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1853 ; GFX7-NEXT: s_endpgm
1854 ;
1855 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1856 ; GFX10-WGP: ; %bb.0: ; %entry
1857 ; GFX10-WGP-NEXT: s_clause 0x1
1858 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1859 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
1860 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1861 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1862 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1863 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1864 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1865 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1866 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
1867 ; GFX10-WGP-NEXT: s_endpgm
1868 ;
1869 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1870 ; GFX10-CU: ; %bb.0: ; %entry
1871 ; GFX10-CU-NEXT: s_clause 0x1
1872 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1873 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
1874 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1875 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
1876 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
1877 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
1878 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1879 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1880 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
1881 ; GFX10-CU-NEXT: s_endpgm
1882 ;
1883 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1884 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1885 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1886 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
1887 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1888 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
1889 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
1890 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
1891 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1892 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1893 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
1894 ; SKIP-CACHE-INV-NEXT: s_endpgm
1895 ;
1896 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1897 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1898 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1899 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1900 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1901 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1902 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1903 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1904 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1905 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
1906 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1907 ;
1908 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1909 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1910 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
1911 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
1912 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1913 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
1914 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
1915 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
1916 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1917 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
1918 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1919 ;
1920 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1921 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1922 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1923 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1924 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1925 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1926 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1927 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1928 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1929 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1930 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1931 ;
1932 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1933 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1934 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
1935 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
1936 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1937 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
1938 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
1939 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
1940 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1941 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
1942 ; GFX940-TGSPLIT-NEXT: s_endpgm
1943 ;
1944 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1945 ; GFX11-WGP: ; %bb.0: ; %entry
1946 ; GFX11-WGP-NEXT: s_clause 0x1
1947 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1948 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
1949 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1950 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1951 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
1952 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1953 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1954 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
1955 ; GFX11-WGP-NEXT: s_endpgm
1956 ;
1957 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
1958 ; GFX11-CU: ; %bb.0: ; %entry
1959 ; GFX11-CU-NEXT: s_clause 0x1
1960 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
1961 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
1962 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1963 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
1964 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1965 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
1966 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1967 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1968 ; GFX11-CU-NEXT: s_endpgm
1969 ptr %out, i32 %in) {
1970 entry:
1971 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
1972 store i32 %val, ptr %out, align 4
1973 ret void
1974 }
1976 define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
1977 ; GFX7-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
1978 ; GFX7: ; %bb.0: ; %entry
1979 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
1980 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1981 ; GFX7-NEXT: s_add_u32 s0, s0, 16
1982 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
1983 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
1984 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
1985 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
1986 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
1987 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
1988 ; GFX7-NEXT: s_endpgm
1989 ;
1990 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
1991 ; GFX10-WGP: ; %bb.0: ; %entry
1992 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
1993 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1994 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
1995 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
1996 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
1997 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
1998 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
1999 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2000 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2001 ; GFX10-WGP-NEXT: s_endpgm
2002 ;
2003 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2004 ; GFX10-CU: ; %bb.0: ; %entry
2005 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2006 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2007 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2008 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2009 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2010 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2011 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2012 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2013 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2014 ; GFX10-CU-NEXT: s_endpgm
2015 ;
2016 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2017 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2018 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2019 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2020 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2021 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2022 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2023 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2024 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2025 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2026 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2027 ; SKIP-CACHE-INV-NEXT: s_endpgm
2028 ;
2029 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2030 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2031 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2032 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2033 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2034 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2035 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2036 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2037 ;
2038 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2039 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2040 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2041 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2042 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2043 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2044 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2045 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2046 ;
2047 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2048 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2049 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2050 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2051 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2052 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2053 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2054 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2055 ;
2056 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2057 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2058 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2059 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2060 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2061 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2062 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2063 ; GFX940-TGSPLIT-NEXT: s_endpgm
2064 ;
2065 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2066 ; GFX11-WGP: ; %bb.0: ; %entry
2067 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2068 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2069 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2070 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2071 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2072 ; GFX11-WGP-NEXT: s_endpgm
2073 ;
2074 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
2075 ; GFX11-CU: ; %bb.0: ; %entry
2076 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2077 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2078 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2079 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2080 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2081 ; GFX11-CU-NEXT: s_endpgm
2082 ptr %out, i32 %in, i32 %old) {
2083 entry:
2084 %gep = getelementptr i32, ptr %out, i32 4
2085 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
2086 ret void
2087 }
2089 define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
2090 ; GFX7-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2091 ; GFX7: ; %bb.0: ; %entry
2092 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2093 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2094 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2095 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2096 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2097 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2098 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2099 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2100 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2101 ; GFX7-NEXT: s_endpgm
2102 ;
2103 ; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2104 ; GFX10-WGP: ; %bb.0: ; %entry
2105 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2106 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2107 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2108 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2109 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2110 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2111 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2112 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2113 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2114 ; GFX10-WGP-NEXT: s_endpgm
2115 ;
2116 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2117 ; GFX10-CU: ; %bb.0: ; %entry
2118 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2119 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2120 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2121 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2122 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2123 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2124 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2125 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2126 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2127 ; GFX10-CU-NEXT: s_endpgm
2128 ;
2129 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2130 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2131 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2132 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2133 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2134 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2135 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2136 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2137 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2138 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2139 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2140 ; SKIP-CACHE-INV-NEXT: s_endpgm
2141 ;
2142 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2143 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2144 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2145 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2146 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2147 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2148 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2149 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2150 ;
2151 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2152 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2153 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2154 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2155 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2156 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2157 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2158 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2159 ;
2160 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2161 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2162 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2163 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2164 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2165 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2166 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2167 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2168 ;
2169 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2170 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2171 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2172 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2173 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2174 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2175 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2176 ; GFX940-TGSPLIT-NEXT: s_endpgm
2177 ;
2178 ; GFX11-WGP-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2179 ; GFX11-WGP: ; %bb.0: ; %entry
2180 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2181 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2182 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2183 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2184 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2185 ; GFX11-WGP-NEXT: s_endpgm
2186 ;
2187 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
2188 ; GFX11-CU: ; %bb.0: ; %entry
2189 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2190 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2191 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2192 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2193 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2194 ; GFX11-CU-NEXT: s_endpgm
2195 ptr %out, i32 %in, i32 %old) {
2196 entry:
2197 %gep = getelementptr i32, ptr %out, i32 4
2198 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
2199 ret void
2200 }
2202 define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
2203 ; GFX7-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2204 ; GFX7: ; %bb.0: ; %entry
2205 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2206 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2207 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2208 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2209 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2210 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2211 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2212 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2213 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2214 ; GFX7-NEXT: s_endpgm
2215 ;
2216 ; GFX10-WGP-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2217 ; GFX10-WGP: ; %bb.0: ; %entry
2218 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2219 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2220 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2221 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2222 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2223 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2224 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2225 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2226 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2227 ; GFX10-WGP-NEXT: s_endpgm
2228 ;
2229 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2230 ; GFX10-CU: ; %bb.0: ; %entry
2231 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2232 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2233 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2234 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2235 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2236 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2237 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2238 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2239 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2240 ; GFX10-CU-NEXT: s_endpgm
2241 ;
2242 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2243 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2244 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2245 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2246 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2247 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2249 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2250 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2251 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2252 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2253 ; SKIP-CACHE-INV-NEXT: s_endpgm
2254 ;
2255 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2256 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2257 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2258 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2259 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2260 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2261 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2262 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2263 ;
2264 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2265 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2266 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2267 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2268 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2269 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2270 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2271 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2272 ;
2273 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2274 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2275 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2276 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2277 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2278 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2279 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2280 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2281 ;
2282 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2283 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2284 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2285 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2286 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2287 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2288 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2289 ; GFX940-TGSPLIT-NEXT: s_endpgm
2290 ;
2291 ; GFX11-WGP-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2292 ; GFX11-WGP: ; %bb.0: ; %entry
2293 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2294 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2295 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2296 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2297 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2298 ; GFX11-WGP-NEXT: s_endpgm
2299 ;
2300 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
2301 ; GFX11-CU: ; %bb.0: ; %entry
2302 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2303 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2304 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2305 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2306 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2307 ; GFX11-CU-NEXT: s_endpgm
2308 ptr %out, i32 %in, i32 %old) {
2309 entry:
2310 %gep = getelementptr i32, ptr %out, i32 4
2311 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
2312 ret void
2313 }
2315 define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
2316 ; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2317 ; GFX7: ; %bb.0: ; %entry
2318 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2319 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2320 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2321 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2322 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2323 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2324 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2325 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2326 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2327 ; GFX7-NEXT: s_endpgm
2328 ;
2329 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2330 ; GFX10-WGP: ; %bb.0: ; %entry
2331 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2332 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2333 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2334 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2335 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2336 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2337 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2338 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2339 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2340 ; GFX10-WGP-NEXT: s_endpgm
2341 ;
2342 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2343 ; GFX10-CU: ; %bb.0: ; %entry
2344 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2345 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2346 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2347 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2348 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2349 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2350 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2351 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2352 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2353 ; GFX10-CU-NEXT: s_endpgm
2354 ;
2355 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2356 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2357 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2358 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2359 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2360 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2361 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2362 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2363 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2364 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2365 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2366 ; SKIP-CACHE-INV-NEXT: s_endpgm
2367 ;
2368 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2369 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2370 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2371 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2372 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2373 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2374 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2375 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2376 ;
2377 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2378 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2379 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2380 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2381 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2382 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2383 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2384 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2385 ;
2386 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2387 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2388 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2389 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2390 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2391 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2392 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2393 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2394 ;
2395 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2396 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2397 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2398 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2399 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2400 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2401 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2402 ; GFX940-TGSPLIT-NEXT: s_endpgm
2403 ;
2404 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2405 ; GFX11-WGP: ; %bb.0: ; %entry
2406 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2407 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2408 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2409 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2410 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2411 ; GFX11-WGP-NEXT: s_endpgm
2412 ;
2413 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
2414 ; GFX11-CU: ; %bb.0: ; %entry
2415 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2416 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2417 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2418 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2419 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2420 ; GFX11-CU-NEXT: s_endpgm
2421 ptr %out, i32 %in, i32 %old) {
2422 entry:
2423 %gep = getelementptr i32, ptr %out, i32 4
2424 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
2425 ret void
2426 }
2428 define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
2429 ; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2430 ; GFX7: ; %bb.0: ; %entry
2431 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2432 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2433 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2434 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2435 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2436 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2437 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2438 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2439 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2440 ; GFX7-NEXT: s_endpgm
2441 ;
2442 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2443 ; GFX10-WGP: ; %bb.0: ; %entry
2444 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2445 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2446 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2447 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2448 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2449 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2450 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2451 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2452 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2453 ; GFX10-WGP-NEXT: s_endpgm
2454 ;
2455 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2456 ; GFX10-CU: ; %bb.0: ; %entry
2457 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2458 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2459 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2460 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2461 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2462 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2463 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2464 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2465 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2466 ; GFX10-CU-NEXT: s_endpgm
2467 ;
2468 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2469 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2470 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2471 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2472 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2473 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2474 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2475 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2476 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2477 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2478 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2479 ; SKIP-CACHE-INV-NEXT: s_endpgm
2480 ;
2481 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2482 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2483 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2484 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2485 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2486 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2487 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2488 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2489 ;
2490 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2491 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2492 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2493 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2494 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2495 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2496 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2497 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2498 ;
2499 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2500 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2501 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2502 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2503 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2504 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2505 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2506 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2507 ;
2508 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2509 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2510 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2511 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2512 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2513 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2514 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2515 ; GFX940-TGSPLIT-NEXT: s_endpgm
2516 ;
2517 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2518 ; GFX11-WGP: ; %bb.0: ; %entry
2519 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2520 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2521 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2522 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2523 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2524 ; GFX11-WGP-NEXT: s_endpgm
2525 ;
2526 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
2527 ; GFX11-CU: ; %bb.0: ; %entry
2528 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2529 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2530 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2531 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2532 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2533 ; GFX11-CU-NEXT: s_endpgm
2534 ptr %out, i32 %in, i32 %old) {
2535 entry:
2536 %gep = getelementptr i32, ptr %out, i32 4
2537 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
2538 ret void
2539 }
2541 define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
2542 ; GFX7-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2543 ; GFX7: ; %bb.0: ; %entry
2544 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2545 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2546 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2547 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2548 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2549 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2550 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2551 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2552 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2553 ; GFX7-NEXT: s_endpgm
2554 ;
2555 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2556 ; GFX10-WGP: ; %bb.0: ; %entry
2557 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2558 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2559 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2560 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2561 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2562 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2563 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2564 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2565 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2566 ; GFX10-WGP-NEXT: s_endpgm
2567 ;
2568 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2569 ; GFX10-CU: ; %bb.0: ; %entry
2570 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2571 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2572 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2573 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2574 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2575 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2576 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2577 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2578 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2579 ; GFX10-CU-NEXT: s_endpgm
2580 ;
2581 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2582 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2583 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2584 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2585 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2586 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2587 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2588 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2591 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2592 ; SKIP-CACHE-INV-NEXT: s_endpgm
2593 ;
2594 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2595 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2596 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2597 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2598 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2599 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2600 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2601 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2602 ;
2603 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2604 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2605 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2606 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2607 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2608 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2609 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2610 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2611 ;
2612 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2613 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2614 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2615 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2616 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2617 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2618 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2619 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2620 ;
2621 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2622 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2623 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2624 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2625 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2626 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2627 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2628 ; GFX940-TGSPLIT-NEXT: s_endpgm
2629 ;
2630 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2631 ; GFX11-WGP: ; %bb.0: ; %entry
2632 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2633 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2634 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2635 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2636 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2637 ; GFX11-WGP-NEXT: s_endpgm
2638 ;
2639 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
2640 ; GFX11-CU: ; %bb.0: ; %entry
2641 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2642 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2643 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2644 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2645 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2646 ; GFX11-CU-NEXT: s_endpgm
2647 ptr %out, i32 %in, i32 %old) {
2648 entry:
2649 %gep = getelementptr i32, ptr %out, i32 4
2650 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic acquire
2651 ret void
2652 }
2654 define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
2655 ; GFX7-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2656 ; GFX7: ; %bb.0: ; %entry
2657 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2658 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2659 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2660 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2661 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2662 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2663 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2664 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2665 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2666 ; GFX7-NEXT: s_endpgm
2667 ;
2668 ; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2669 ; GFX10-WGP: ; %bb.0: ; %entry
2670 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2671 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2672 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2673 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2674 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2675 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2676 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2677 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2678 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2679 ; GFX10-WGP-NEXT: s_endpgm
2680 ;
2681 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2682 ; GFX10-CU: ; %bb.0: ; %entry
2683 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2684 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2685 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2686 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2687 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2688 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2689 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2690 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2691 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2692 ; GFX10-CU-NEXT: s_endpgm
2693 ;
2694 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2695 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2696 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2697 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2698 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2699 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2700 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2701 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2702 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2703 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2704 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2705 ; SKIP-CACHE-INV-NEXT: s_endpgm
2706 ;
2707 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2708 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2709 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2710 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2711 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2712 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2713 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2714 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2715 ;
2716 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2717 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2718 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2719 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2720 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2721 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2722 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2723 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2724 ;
2725 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2726 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2727 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2728 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2729 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2730 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2731 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2732 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2733 ;
2734 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2735 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2736 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2737 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2738 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2739 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2740 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2741 ; GFX940-TGSPLIT-NEXT: s_endpgm
2742 ;
2743 ; GFX11-WGP-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2744 ; GFX11-WGP: ; %bb.0: ; %entry
2745 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2746 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2747 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2748 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2749 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2750 ; GFX11-WGP-NEXT: s_endpgm
2751 ;
2752 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
2753 ; GFX11-CU: ; %bb.0: ; %entry
2754 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2755 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2756 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2757 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2758 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2759 ; GFX11-CU-NEXT: s_endpgm
2760 ptr %out, i32 %in, i32 %old) {
2761 entry:
2762 %gep = getelementptr i32, ptr %out, i32 4
2763 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
2764 ret void
2765 }
2767 define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
2768 ; GFX7-LABEL: flat_wavefront_release_acquire_cmpxchg:
2769 ; GFX7: ; %bb.0: ; %entry
2770 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2771 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2772 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2773 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2774 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2775 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2776 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2777 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2778 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2779 ; GFX7-NEXT: s_endpgm
2780 ;
2781 ; GFX10-WGP-LABEL: flat_wavefront_release_acquire_cmpxchg:
2782 ; GFX10-WGP: ; %bb.0: ; %entry
2783 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2784 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2785 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2786 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2788 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2789 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2791 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2792 ; GFX10-WGP-NEXT: s_endpgm
2793 ;
2794 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
2795 ; GFX10-CU: ; %bb.0: ; %entry
2796 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2797 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2798 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2799 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2800 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2801 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2802 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2803 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2804 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2805 ; GFX10-CU-NEXT: s_endpgm
2806 ;
2807 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_acquire_cmpxchg:
2808 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2809 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2810 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2811 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2812 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2813 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2814 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2815 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2816 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2817 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2818 ; SKIP-CACHE-INV-NEXT: s_endpgm
2819 ;
2820 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
2821 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2822 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2823 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2824 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2825 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2826 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2827 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2828 ;
2829 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
2830 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2831 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2832 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2833 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2834 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2835 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2836 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2837 ;
2838 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
2839 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2840 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2841 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2842 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2843 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2844 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2845 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2846 ;
2847 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_acquire_cmpxchg:
2848 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2849 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2850 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2851 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2852 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2853 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2854 ; GFX940-TGSPLIT-NEXT: s_endpgm
2855 ;
2856 ; GFX11-WGP-LABEL: flat_wavefront_release_acquire_cmpxchg:
2857 ; GFX11-WGP: ; %bb.0: ; %entry
2858 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2859 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2860 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2861 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2862 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2863 ; GFX11-WGP-NEXT: s_endpgm
2864 ;
2865 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
2866 ; GFX11-CU: ; %bb.0: ; %entry
2867 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2868 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2869 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2870 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2871 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2872 ; GFX11-CU-NEXT: s_endpgm
2873 ptr %out, i32 %in, i32 %old) {
2874 entry:
2875 %gep = getelementptr i32, ptr %out, i32 4
2876 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
2877 ret void
2878 }
2880 define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
2881 ; GFX7-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2882 ; GFX7: ; %bb.0: ; %entry
2883 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2884 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2885 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2886 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
2887 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
2888 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
2889 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
2890 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
2891 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2892 ; GFX7-NEXT: s_endpgm
2893 ;
2894 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2895 ; GFX10-WGP: ; %bb.0: ; %entry
2896 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2897 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2898 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
2899 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
2900 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
2901 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
2902 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
2903 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
2904 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2905 ; GFX10-WGP-NEXT: s_endpgm
2906 ;
2907 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2908 ; GFX10-CU: ; %bb.0: ; %entry
2909 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2910 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2911 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
2912 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
2913 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
2914 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
2915 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
2916 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
2917 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2918 ; GFX10-CU-NEXT: s_endpgm
2919 ;
2920 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2921 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2922 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2923 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2924 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
2925 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
2926 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
2927 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
2928 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
2929 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
2930 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
2931 ; SKIP-CACHE-INV-NEXT: s_endpgm
2932 ;
2933 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2934 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2935 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2936 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2937 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2938 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2939 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2940 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2941 ;
2942 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2943 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2944 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2945 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2946 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
2947 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
2948 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2949 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2950 ;
2951 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2952 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2953 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2954 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2955 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2956 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2957 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2958 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2959 ;
2960 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2961 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2962 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
2963 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2964 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
2965 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
2966 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
2967 ; GFX940-TGSPLIT-NEXT: s_endpgm
2968 ;
2969 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2970 ; GFX11-WGP: ; %bb.0: ; %entry
2971 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2972 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2973 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2974 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2975 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2976 ; GFX11-WGP-NEXT: s_endpgm
2977 ;
2978 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
2979 ; GFX11-CU: ; %bb.0: ; %entry
2980 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
2981 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2982 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
2983 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
2984 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
2985 ; GFX11-CU-NEXT: s_endpgm
2986 ptr %out, i32 %in, i32 %old) {
2987 entry:
2988 %gep = getelementptr i32, ptr %out, i32 4
2989 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
2990 ret void
2991 }
2993 define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
2994 ; GFX7-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
2995 ; GFX7: ; %bb.0: ; %entry
2996 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
2997 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2998 ; GFX7-NEXT: s_add_u32 s0, s0, 16
2999 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3000 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3001 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3002 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3003 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3004 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3005 ; GFX7-NEXT: s_endpgm
3006 ;
3007 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3008 ; GFX10-WGP: ; %bb.0: ; %entry
3009 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3010 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3011 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3012 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3013 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3014 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3015 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3016 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3017 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3018 ; GFX10-WGP-NEXT: s_endpgm
3019 ;
3020 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3021 ; GFX10-CU: ; %bb.0: ; %entry
3022 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3023 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3024 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3025 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3026 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3027 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3028 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3029 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3030 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3031 ; GFX10-CU-NEXT: s_endpgm
3032 ;
3033 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3034 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3035 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3036 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3037 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3038 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3039 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3040 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3041 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3042 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3043 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3044 ; SKIP-CACHE-INV-NEXT: s_endpgm
3045 ;
3046 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3047 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3048 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3049 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3050 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3051 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3052 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3053 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3054 ;
3055 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3056 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3057 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3058 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3059 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3060 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3061 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3062 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3063 ;
3064 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3065 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3066 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3067 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3068 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3069 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3070 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3071 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3072 ;
3073 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3074 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3075 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3076 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3077 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3078 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3079 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3080 ; GFX940-TGSPLIT-NEXT: s_endpgm
3081 ;
3082 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3083 ; GFX11-WGP: ; %bb.0: ; %entry
3084 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3085 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3086 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3087 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3088 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3089 ; GFX11-WGP-NEXT: s_endpgm
3090 ;
3091 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
3092 ; GFX11-CU: ; %bb.0: ; %entry
3093 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3094 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3095 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3096 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3097 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3098 ; GFX11-CU-NEXT: s_endpgm
3099 ptr %out, i32 %in, i32 %old) {
3100 entry:
3101 %gep = getelementptr i32, ptr %out, i32 4
3102 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
3103 ret void
3104 }
3106 define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
3107 ; GFX7-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3108 ; GFX7: ; %bb.0: ; %entry
3109 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3110 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3111 ; GFX7-NEXT: s_add_u32 s0, s0, 16
3112 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3113 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3114 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3115 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3116 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3117 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3118 ; GFX7-NEXT: s_endpgm
3119 ;
3120 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3121 ; GFX10-WGP: ; %bb.0: ; %entry
3122 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3123 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3124 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3125 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3126 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3127 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3128 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3129 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3130 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3131 ; GFX10-WGP-NEXT: s_endpgm
3132 ;
3133 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3134 ; GFX10-CU: ; %bb.0: ; %entry
3135 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3136 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3137 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3138 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3139 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3140 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3141 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3142 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3143 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3144 ; GFX10-CU-NEXT: s_endpgm
3145 ;
3146 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3147 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3148 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3149 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3150 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3151 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3152 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3153 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3154 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3155 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3156 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3157 ; SKIP-CACHE-INV-NEXT: s_endpgm
3158 ;
3159 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3160 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3161 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3162 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3163 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3164 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3165 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3166 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3167 ;
3168 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3169 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3170 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3171 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3172 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3173 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3174 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3175 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3176 ;
3177 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3178 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3179 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3180 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3181 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3182 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3183 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3184 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3185 ;
3186 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3187 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3188 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3189 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3190 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3191 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3192 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3193 ; GFX940-TGSPLIT-NEXT: s_endpgm
3194 ;
3195 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3196 ; GFX11-WGP: ; %bb.0: ; %entry
3197 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3198 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3199 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3200 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3201 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3202 ; GFX11-WGP-NEXT: s_endpgm
3203 ;
3204 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
3205 ; GFX11-CU: ; %bb.0: ; %entry
3206 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3207 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3208 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3209 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3210 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3211 ; GFX11-CU-NEXT: s_endpgm
3212 ptr %out, i32 %in, i32 %old) {
3213 entry:
3214 %gep = getelementptr i32, ptr %out, i32 4
3215 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic seq_cst
3216 ret void
3217 }
3219 define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
3220 ; GFX7-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3221 ; GFX7: ; %bb.0: ; %entry
3222 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3223 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3224 ; GFX7-NEXT: s_add_u32 s0, s0, 16
3225 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3226 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3227 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3228 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3229 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3230 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3231 ; GFX7-NEXT: s_endpgm
3232 ;
3233 ; GFX10-WGP-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3234 ; GFX10-WGP: ; %bb.0: ; %entry
3235 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3236 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3237 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3238 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3239 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3240 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3241 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3242 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3243 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3244 ; GFX10-WGP-NEXT: s_endpgm
3245 ;
3246 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3247 ; GFX10-CU: ; %bb.0: ; %entry
3248 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3249 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3250 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3251 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3252 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3253 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3254 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3255 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3256 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3257 ; GFX10-CU-NEXT: s_endpgm
3258 ;
3259 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3260 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3261 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3262 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3263 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3264 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3265 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3266 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3267 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3268 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3269 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3270 ; SKIP-CACHE-INV-NEXT: s_endpgm
3271 ;
3272 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3273 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3274 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3275 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3276 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3277 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3278 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3279 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3280 ;
3281 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3282 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3283 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3284 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3285 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3286 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3287 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3288 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3289 ;
3290 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3291 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3292 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3293 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3294 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3295 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3296 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3297 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3298 ;
3299 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3300 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3301 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3302 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3303 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3304 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3305 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3306 ; GFX940-TGSPLIT-NEXT: s_endpgm
3307 ;
3308 ; GFX11-WGP-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3309 ; GFX11-WGP: ; %bb.0: ; %entry
3310 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3311 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3312 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3313 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3314 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3315 ; GFX11-WGP-NEXT: s_endpgm
3316 ;
3317 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
3318 ; GFX11-CU: ; %bb.0: ; %entry
3319 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3320 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3321 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3322 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3323 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3324 ; GFX11-CU-NEXT: s_endpgm
3325 ptr %out, i32 %in, i32 %old) {
3326 entry:
3327 %gep = getelementptr i32, ptr %out, i32 4
3328 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire seq_cst
3329 ret void
3330 }
3332 define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
3333 ; GFX7-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3334 ; GFX7: ; %bb.0: ; %entry
3335 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3336 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3337 ; GFX7-NEXT: s_add_u32 s0, s0, 16
3338 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3339 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3340 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3341 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3342 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3343 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3344 ; GFX7-NEXT: s_endpgm
3345 ;
3346 ; GFX10-WGP-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3347 ; GFX10-WGP: ; %bb.0: ; %entry
3348 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3349 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3350 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3351 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3352 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3353 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3354 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3355 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3356 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3357 ; GFX10-WGP-NEXT: s_endpgm
3358 ;
3359 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3360 ; GFX10-CU: ; %bb.0: ; %entry
3361 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3362 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3363 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3364 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3365 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3366 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3367 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3368 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3369 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3370 ; GFX10-CU-NEXT: s_endpgm
3371 ;
3372 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3373 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3374 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3375 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3376 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3377 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3378 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3379 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3380 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3381 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3382 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3383 ; SKIP-CACHE-INV-NEXT: s_endpgm
3384 ;
3385 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3386 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3387 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3388 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3389 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3390 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3391 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3392 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3393 ;
3394 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3395 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3396 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3397 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3398 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3399 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3400 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3401 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3402 ;
3403 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3404 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3405 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3406 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3407 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3408 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3409 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3410 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3411 ;
3412 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3413 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3414 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3415 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3416 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3417 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3418 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3419 ; GFX940-TGSPLIT-NEXT: s_endpgm
3420 ;
3421 ; GFX11-WGP-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3422 ; GFX11-WGP: ; %bb.0: ; %entry
3423 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3424 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3425 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3426 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3427 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3428 ; GFX11-WGP-NEXT: s_endpgm
3429 ;
3430 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
3431 ; GFX11-CU: ; %bb.0: ; %entry
3432 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3433 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3434 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3435 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3436 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3437 ; GFX11-CU-NEXT: s_endpgm
3438 ptr %out, i32 %in, i32 %old) {
3439 entry:
3440 %gep = getelementptr i32, ptr %out, i32 4
3441 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release seq_cst
3442 ret void
3443 }
3445 define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
3446 ; GFX7-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3447 ; GFX7: ; %bb.0: ; %entry
3448 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3449 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3450 ; GFX7-NEXT: s_add_u32 s0, s0, 16
3451 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3452 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3453 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3454 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3455 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3456 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3457 ; GFX7-NEXT: s_endpgm
3458 ;
3459 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3460 ; GFX10-WGP: ; %bb.0: ; %entry
3461 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3462 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3463 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3464 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3465 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3466 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3467 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3468 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3469 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3470 ; GFX10-WGP-NEXT: s_endpgm
3471 ;
3472 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3473 ; GFX10-CU: ; %bb.0: ; %entry
3474 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3475 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3476 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3477 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3478 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3479 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3480 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3481 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3482 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3483 ; GFX10-CU-NEXT: s_endpgm
3484 ;
3485 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3486 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3487 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3488 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3489 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3490 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3491 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3492 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3493 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3494 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3495 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3496 ; SKIP-CACHE-INV-NEXT: s_endpgm
3497 ;
3498 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3499 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3500 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3501 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3502 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3503 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3504 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3505 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3506 ;
3507 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3508 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3509 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3510 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3511 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3512 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3513 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3514 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3515 ;
3516 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3517 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3518 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3519 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3520 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3521 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3522 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3523 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3524 ;
3525 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3526 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3527 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3528 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3529 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3530 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3531 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3532 ; GFX940-TGSPLIT-NEXT: s_endpgm
3533 ;
3534 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3535 ; GFX11-WGP: ; %bb.0: ; %entry
3536 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3537 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3538 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3539 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3540 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3541 ; GFX11-WGP-NEXT: s_endpgm
3542 ;
3543 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
3544 ; GFX11-CU: ; %bb.0: ; %entry
3545 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3546 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3547 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3548 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3549 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3550 ; GFX11-CU-NEXT: s_endpgm
3551 ptr %out, i32 %in, i32 %old) {
3552 entry:
3553 %gep = getelementptr i32, ptr %out, i32 4
3554 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel seq_cst
3555 ret void
3556 }
3558 define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
3559 ; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3560 ; GFX7: ; %bb.0: ; %entry
3561 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3562 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3563 ; GFX7-NEXT: s_add_u32 s0, s0, 16
3564 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
3565 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3566 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3567 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3568 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3569 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3570 ; GFX7-NEXT: s_endpgm
3571 ;
3572 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3573 ; GFX10-WGP: ; %bb.0: ; %entry
3574 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3575 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3576 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
3577 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
3578 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3579 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3580 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3581 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3582 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3583 ; GFX10-WGP-NEXT: s_endpgm
3584 ;
3585 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3586 ; GFX10-CU: ; %bb.0: ; %entry
3587 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3588 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3589 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
3590 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
3591 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3592 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3593 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3594 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3595 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3596 ; GFX10-CU-NEXT: s_endpgm
3597 ;
3598 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3599 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3600 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3601 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3602 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
3603 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
3604 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3605 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3606 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3607 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3608 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3609 ; SKIP-CACHE-INV-NEXT: s_endpgm
3610 ;
3611 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3612 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3613 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3614 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3615 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3616 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3617 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3618 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3619 ;
3620 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3621 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3622 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3623 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3624 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3625 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3626 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3627 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3628 ;
3629 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3630 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3631 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3632 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3633 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3634 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3635 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3636 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3637 ;
3638 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3639 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3640 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3641 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3642 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3643 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3644 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
3645 ; GFX940-TGSPLIT-NEXT: s_endpgm
3646 ;
3647 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3648 ; GFX11-WGP: ; %bb.0: ; %entry
3649 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3650 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3651 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3652 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3653 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3654 ; GFX11-WGP-NEXT: s_endpgm
3655 ;
3656 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
3657 ; GFX11-CU: ; %bb.0: ; %entry
3658 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3659 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3660 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3661 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3662 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
3663 ; GFX11-CU-NEXT: s_endpgm
3664 ptr %out, i32 %in, i32 %old) {
3665 entry:
3666 %gep = getelementptr i32, ptr %out, i32 4
3667 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
3668 ret void
3669 }
3671 define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
3672 ; GFX7-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3673 ; GFX7: ; %bb.0: ; %entry
3674 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3675 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3676 ; GFX7-NEXT: s_add_u32 s4, s0, 16
3677 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
3678 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3679 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3680 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3681 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3682 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3683 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3684 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3685 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3686 ; GFX7-NEXT: flat_store_dword v[0:1], v2
3687 ; GFX7-NEXT: s_endpgm
3688 ;
3689 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3690 ; GFX10-WGP: ; %bb.0: ; %entry
3691 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3692 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3693 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
3694 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
3695 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3696 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3697 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3698 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3699 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3700 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3701 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3702 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3703 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
3704 ; GFX10-WGP-NEXT: s_endpgm
3705 ;
3706 ; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3707 ; GFX10-CU: ; %bb.0: ; %entry
3708 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3709 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3710 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
3711 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
3712 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3713 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3714 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3715 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3716 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3717 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3718 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3719 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3720 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
3721 ; GFX10-CU-NEXT: s_endpgm
3722 ;
3723 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3724 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3725 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3726 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3727 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
3728 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
3729 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
3730 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
3732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3733 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3734 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3735 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3736 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3737 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
3738 ; SKIP-CACHE-INV-NEXT: s_endpgm
3739 ;
3740 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3741 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3742 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3743 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3744 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3745 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3746 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3747 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3748 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
3749 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3750 ;
3751 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3752 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3753 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3754 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3755 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3756 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3757 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3758 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3759 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
3760 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3761 ;
3762 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3763 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3764 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3765 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3766 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3767 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3768 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3769 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3770 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
3771 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3772 ;
3773 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3774 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3775 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3776 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3777 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3778 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3779 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3780 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3781 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
3782 ; GFX940-TGSPLIT-NEXT: s_endpgm
3783 ;
3784 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3785 ; GFX11-WGP: ; %bb.0: ; %entry
3786 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3787 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3788 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3789 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3790 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3791 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3792 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
3793 ; GFX11-WGP-NEXT: s_endpgm
3794 ;
3795 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
3796 ; GFX11-CU: ; %bb.0: ; %entry
3797 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3798 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3799 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3800 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3801 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3802 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3803 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
3804 ; GFX11-CU-NEXT: s_endpgm
3805 ptr %out, i32 %in, i32 %old) {
3806 entry:
3807 %gep = getelementptr i32, ptr %out, i32 4
3808 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
3809 %val0 = extractvalue { i32, i1 } %val, 0
3810 store i32 %val0, ptr %out, align 4
3811 ret void
3812 }
3814 define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
3815 ; GFX7-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3816 ; GFX7: ; %bb.0: ; %entry
3817 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3818 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3819 ; GFX7-NEXT: s_add_u32 s4, s0, 16
3820 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
3821 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3822 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3823 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3824 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3825 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3826 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3827 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3828 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3829 ; GFX7-NEXT: flat_store_dword v[0:1], v2
3830 ; GFX7-NEXT: s_endpgm
3831 ;
3832 ; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3833 ; GFX10-WGP: ; %bb.0: ; %entry
3834 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3835 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3836 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
3837 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
3838 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3839 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3840 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3841 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3842 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3843 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3844 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3845 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3846 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
3847 ; GFX10-WGP-NEXT: s_endpgm
3848 ;
3849 ; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3850 ; GFX10-CU: ; %bb.0: ; %entry
3851 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3852 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3853 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
3854 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
3855 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3856 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
3857 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
3858 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
3859 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3860 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
3861 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
3862 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3863 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
3864 ; GFX10-CU-NEXT: s_endpgm
3865 ;
3866 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3867 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3868 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3869 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3870 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
3871 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
3872 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
3873 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
3874 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
3875 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
3876 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3877 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
3878 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
3879 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3880 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
3881 ; SKIP-CACHE-INV-NEXT: s_endpgm
3882 ;
3883 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3884 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3885 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3886 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3887 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3888 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3889 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3890 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3891 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
3892 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3893 ;
3894 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3895 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3896 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3897 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3898 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
3899 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
3900 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
3901 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3902 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
3903 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3904 ;
3905 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3906 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3907 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3908 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3909 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3910 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3911 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3912 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3913 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
3914 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3915 ;
3916 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3917 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3918 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
3919 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3920 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
3921 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
3922 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
3923 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3924 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
3925 ; GFX940-TGSPLIT-NEXT: s_endpgm
3926 ;
3927 ; GFX11-WGP-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3928 ; GFX11-WGP: ; %bb.0: ; %entry
3929 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3930 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3931 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3932 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3933 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3934 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3935 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
3936 ; GFX11-WGP-NEXT: s_endpgm
3937 ;
3938 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
3939 ; GFX11-CU: ; %bb.0: ; %entry
3940 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
3941 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3942 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
3943 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
3944 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
3945 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3946 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
3947 ; GFX11-CU-NEXT: s_endpgm
3948 ptr %out, i32 %in, i32 %old) {
3949 entry:
3950 %gep = getelementptr i32, ptr %out, i32 4
3951 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
3952 %val0 = extractvalue { i32, i1 } %val, 0
3953 store i32 %val0, ptr %out, align 4
3954 ret void
3955 }
3957 define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
3958 ; GFX7-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
3959 ; GFX7: ; %bb.0: ; %entry
3960 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3961 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3962 ; GFX7-NEXT: s_add_u32 s4, s0, 16
3963 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
3964 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3965 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
3966 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3967 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
3968 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3969 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
3970 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
3971 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3972 ; GFX7-NEXT: flat_store_dword v[0:1], v2
3973 ; GFX7-NEXT: s_endpgm
3974 ;
3975 ; GFX10-WGP-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
3976 ; GFX10-WGP: ; %bb.0: ; %entry
3977 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3978 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3979 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
3980 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
3981 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
3982 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
3983 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
3984 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
3985 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3986 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
3987 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
3988 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3989 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
3990 ; GFX10-WGP-NEXT: s_endpgm
3991 ;
3992 ; GFX10-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
3993 ; GFX10-CU: ; %bb.0: ; %entry
3994 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
3995 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3996 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
3997 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
3998 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
3999 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4000 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4001 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4002 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4003 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4004 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4005 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4006 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4007 ; GFX10-CU-NEXT: s_endpgm
4008 ;
4009 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4010 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4011 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4012 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4013 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4014 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4015 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4016 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4017 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4018 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4019 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4020 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4021 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4022 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4023 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4024 ; SKIP-CACHE-INV-NEXT: s_endpgm
4025 ;
4026 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4027 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4028 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4029 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4030 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4031 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4032 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4033 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4034 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4035 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4036 ;
4037 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4038 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4039 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4040 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4041 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4042 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4043 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4044 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4045 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4046 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4047 ;
4048 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4049 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4050 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4051 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4052 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4053 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4054 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4055 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4056 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4057 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4058 ;
4059 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4060 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4061 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4062 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4063 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4064 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4065 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4066 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4067 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4068 ; GFX940-TGSPLIT-NEXT: s_endpgm
4069 ;
4070 ; GFX11-WGP-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4071 ; GFX11-WGP: ; %bb.0: ; %entry
4072 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4073 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4074 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4075 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4076 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4077 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4078 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4079 ; GFX11-WGP-NEXT: s_endpgm
4080 ;
4081 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
4082 ; GFX11-CU: ; %bb.0: ; %entry
4083 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4084 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4085 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4086 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4087 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4088 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4089 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4090 ; GFX11-CU-NEXT: s_endpgm
4091 ptr %out, i32 %in, i32 %old) {
4092 entry:
4093 %gep = getelementptr i32, ptr %out, i32 4
4094 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
4095 %val0 = extractvalue { i32, i1 } %val, 0
4096 store i32 %val0, ptr %out, align 4
4097 ret void
4098 }
4100 define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
4101 ; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4102 ; GFX7: ; %bb.0: ; %entry
4103 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4104 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4105 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4106 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4107 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4108 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4109 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4110 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4111 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4112 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4113 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4114 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4115 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4116 ; GFX7-NEXT: s_endpgm
4117 ;
4118 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4119 ; GFX10-WGP: ; %bb.0: ; %entry
4120 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4121 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4122 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4123 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4124 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4125 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4126 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4127 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4128 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4129 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4130 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4131 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4132 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4133 ; GFX10-WGP-NEXT: s_endpgm
4134 ;
4135 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4136 ; GFX10-CU: ; %bb.0: ; %entry
4137 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4138 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4139 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4140 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4141 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4142 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4143 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4144 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4145 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4146 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4147 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4148 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4149 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4150 ; GFX10-CU-NEXT: s_endpgm
4151 ;
4152 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4153 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4154 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4155 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4156 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4157 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4158 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4159 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4160 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4161 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4162 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4163 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4164 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4165 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4166 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4167 ; SKIP-CACHE-INV-NEXT: s_endpgm
4168 ;
4169 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4170 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4171 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4172 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4173 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4174 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4175 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4176 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4177 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4178 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4179 ;
4180 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4181 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4182 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4183 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4184 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4185 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4186 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4187 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4188 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4189 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4190 ;
4191 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4192 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4193 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4194 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4195 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4196 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4197 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4198 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4199 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4200 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4201 ;
4202 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4203 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4204 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4205 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4206 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4207 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4208 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4209 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4210 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4211 ; GFX940-TGSPLIT-NEXT: s_endpgm
4212 ;
4213 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4214 ; GFX11-WGP: ; %bb.0: ; %entry
4215 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4216 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4217 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4218 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4219 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4220 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4221 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4222 ; GFX11-WGP-NEXT: s_endpgm
4223 ;
4224 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
4225 ; GFX11-CU: ; %bb.0: ; %entry
4226 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4227 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4228 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4229 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4230 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4231 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4232 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4233 ; GFX11-CU-NEXT: s_endpgm
4234 ptr %out, i32 %in, i32 %old) {
4235 entry:
4236 %gep = getelementptr i32, ptr %out, i32 4
4237 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
4238 %val0 = extractvalue { i32, i1 } %val, 0
4239 store i32 %val0, ptr %out, align 4
4240 ret void
4241 }
4243 define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
4244 ; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4245 ; GFX7: ; %bb.0: ; %entry
4246 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4247 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4248 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4249 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4250 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4251 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4252 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4253 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4254 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4255 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4256 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4257 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4258 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4259 ; GFX7-NEXT: s_endpgm
4260 ;
4261 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4262 ; GFX10-WGP: ; %bb.0: ; %entry
4263 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4264 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4265 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4266 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4267 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4268 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4270 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4271 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4272 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4273 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4274 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4275 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4276 ; GFX10-WGP-NEXT: s_endpgm
4277 ;
4278 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4279 ; GFX10-CU: ; %bb.0: ; %entry
4280 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4281 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4282 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4283 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4284 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4285 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4286 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4287 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4288 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4289 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4290 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4291 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4292 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4293 ; GFX10-CU-NEXT: s_endpgm
4294 ;
4295 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4296 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4297 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4298 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4299 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4300 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4301 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4302 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4304 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4305 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4308 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4309 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4310 ; SKIP-CACHE-INV-NEXT: s_endpgm
4311 ;
4312 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4313 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4314 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4315 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4316 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4317 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4318 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4319 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4320 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4321 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4322 ;
4323 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4324 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4325 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4326 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4327 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4328 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4329 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4330 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4331 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4332 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4333 ;
4334 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4335 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4336 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4337 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4338 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4339 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4340 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4341 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4342 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4343 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4344 ;
4345 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4346 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4347 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4348 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4349 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4350 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4351 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4352 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4353 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4354 ; GFX940-TGSPLIT-NEXT: s_endpgm
4355 ;
4356 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4357 ; GFX11-WGP: ; %bb.0: ; %entry
4358 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4359 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4360 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4361 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4362 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4363 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4364 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4365 ; GFX11-WGP-NEXT: s_endpgm
4366 ;
4367 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
4368 ; GFX11-CU: ; %bb.0: ; %entry
4369 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4370 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4371 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4372 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4373 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4374 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4375 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4376 ; GFX11-CU-NEXT: s_endpgm
4377 ptr %out, i32 %in, i32 %old) {
4378 entry:
4379 %gep = getelementptr i32, ptr %out, i32 4
4380 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
4381 %val0 = extractvalue { i32, i1 } %val, 0
4382 store i32 %val0, ptr %out, align 4
4383 ret void
4384 }
4386 define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
4387 ; GFX7-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4388 ; GFX7: ; %bb.0: ; %entry
4389 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4390 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4391 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4392 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4393 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4394 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4395 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4396 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4397 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4398 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4399 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4400 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4401 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4402 ; GFX7-NEXT: s_endpgm
4403 ;
4404 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4405 ; GFX10-WGP: ; %bb.0: ; %entry
4406 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4407 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4408 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4409 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4410 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4411 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4412 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4413 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4414 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4415 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4416 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4417 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4418 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4419 ; GFX10-WGP-NEXT: s_endpgm
4420 ;
4421 ; GFX10-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4422 ; GFX10-CU: ; %bb.0: ; %entry
4423 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4424 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4425 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4426 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4427 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4428 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4429 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4430 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4431 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4432 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4433 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4434 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4435 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4436 ; GFX10-CU-NEXT: s_endpgm
4437 ;
4438 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4439 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4440 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4441 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4442 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4443 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4444 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4445 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4446 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4447 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4448 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4449 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4450 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4451 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4452 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4453 ; SKIP-CACHE-INV-NEXT: s_endpgm
4454 ;
4455 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4456 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4457 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4458 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4459 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4460 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4461 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4462 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4463 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4464 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4465 ;
4466 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4467 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4468 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4469 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4470 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4471 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4472 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4473 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4474 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4475 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4476 ;
4477 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4478 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4479 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4480 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4481 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4482 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4483 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4484 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4485 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4486 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4487 ;
4488 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4489 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4490 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4491 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4492 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4493 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4494 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4495 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4496 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4497 ; GFX940-TGSPLIT-NEXT: s_endpgm
4498 ;
4499 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4500 ; GFX11-WGP: ; %bb.0: ; %entry
4501 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4502 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4503 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4504 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4505 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4506 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4507 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4508 ; GFX11-WGP-NEXT: s_endpgm
4509 ;
4510 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
4511 ; GFX11-CU: ; %bb.0: ; %entry
4512 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4513 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4514 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4515 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4516 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4517 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4518 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4519 ; GFX11-CU-NEXT: s_endpgm
4520 ptr %out, i32 %in, i32 %old) {
4521 entry:
4522 %gep = getelementptr i32, ptr %out, i32 4
4523 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic acquire
4524 %val0 = extractvalue { i32, i1 } %val, 0
4525 store i32 %val0, ptr %out, align 4
4526 ret void
4527 }
4529 define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
4530 ; GFX7-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4531 ; GFX7: ; %bb.0: ; %entry
4532 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4533 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4534 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4535 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4536 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4537 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4538 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4539 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4540 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4541 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4542 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4543 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4544 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4545 ; GFX7-NEXT: s_endpgm
4546 ;
4547 ; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4548 ; GFX10-WGP: ; %bb.0: ; %entry
4549 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4550 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4551 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4552 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4553 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4554 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4555 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4556 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4557 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4558 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4559 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4560 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4561 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4562 ; GFX10-WGP-NEXT: s_endpgm
4563 ;
4564 ; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4565 ; GFX10-CU: ; %bb.0: ; %entry
4566 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4567 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4568 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4569 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4570 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4571 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4572 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4573 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4574 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4575 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4576 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4577 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4578 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4579 ; GFX10-CU-NEXT: s_endpgm
4580 ;
4581 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4582 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4583 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4584 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4585 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4586 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4587 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4588 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4591 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4592 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4593 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4594 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4595 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4596 ; SKIP-CACHE-INV-NEXT: s_endpgm
4597 ;
4598 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4599 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4600 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4601 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4602 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4603 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4604 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4605 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4606 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4607 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4608 ;
4609 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4610 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4611 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4612 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4613 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4614 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4615 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4616 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4617 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4618 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4619 ;
4620 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4621 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4622 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4623 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4624 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4625 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4626 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4627 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4628 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4629 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4630 ;
4631 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4632 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4633 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4634 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4635 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4636 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4637 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4638 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4639 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4640 ; GFX940-TGSPLIT-NEXT: s_endpgm
4641 ;
4642 ; GFX11-WGP-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4643 ; GFX11-WGP: ; %bb.0: ; %entry
4644 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4645 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4646 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4647 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4648 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4649 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4650 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4651 ; GFX11-WGP-NEXT: s_endpgm
4652 ;
4653 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
4654 ; GFX11-CU: ; %bb.0: ; %entry
4655 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4656 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4657 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4658 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4659 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4660 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4661 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4662 ; GFX11-CU-NEXT: s_endpgm
4663 ptr %out, i32 %in, i32 %old) {
4664 entry:
4665 %gep = getelementptr i32, ptr %out, i32 4
4666 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
4667 %val0 = extractvalue { i32, i1 } %val, 0
4668 store i32 %val0, ptr %out, align 4
4669 ret void
4670 }
4672 define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
4673 ; GFX7-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4674 ; GFX7: ; %bb.0: ; %entry
4675 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4676 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4677 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4678 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4679 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4680 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4681 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4682 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4683 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4684 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4685 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4686 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4687 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4688 ; GFX7-NEXT: s_endpgm
4689 ;
4690 ; GFX10-WGP-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4691 ; GFX10-WGP: ; %bb.0: ; %entry
4692 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4693 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4694 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4695 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4696 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4697 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4698 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4699 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4700 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4701 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4702 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4703 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4704 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4705 ; GFX10-WGP-NEXT: s_endpgm
4706 ;
4707 ; GFX10-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4708 ; GFX10-CU: ; %bb.0: ; %entry
4709 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4710 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4711 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4712 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4713 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4714 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4715 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4716 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4717 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4718 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4719 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4720 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4721 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4722 ; GFX10-CU-NEXT: s_endpgm
4723 ;
4724 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4725 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4726 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4727 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4728 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4729 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4730 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4733 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4734 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4735 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4737 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4738 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4739 ; SKIP-CACHE-INV-NEXT: s_endpgm
4740 ;
4741 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4742 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4743 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4744 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4745 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4746 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4747 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4748 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4749 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4750 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4751 ;
4752 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4753 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4754 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4755 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4756 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4757 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4758 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4759 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4760 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4761 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4762 ;
4763 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4764 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4765 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4766 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4767 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4768 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4769 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4770 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4771 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4772 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4773 ;
4774 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4775 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4776 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4777 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4778 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4779 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4780 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4781 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4782 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4783 ; GFX940-TGSPLIT-NEXT: s_endpgm
4784 ;
4785 ; GFX11-WGP-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4786 ; GFX11-WGP: ; %bb.0: ; %entry
4787 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4788 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4789 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4790 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4791 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4792 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4793 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4794 ; GFX11-WGP-NEXT: s_endpgm
4795 ;
4796 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
4797 ; GFX11-CU: ; %bb.0: ; %entry
4798 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4799 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4800 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4801 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4802 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4803 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4804 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4805 ; GFX11-CU-NEXT: s_endpgm
4806 ptr %out, i32 %in, i32 %old) {
4807 entry:
4808 %gep = getelementptr i32, ptr %out, i32 4
4809 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
4810 %val0 = extractvalue { i32, i1 } %val, 0
4811 store i32 %val0, ptr %out, align 4
4812 ret void
4813 }
4815 define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
4816 ; GFX7-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4817 ; GFX7: ; %bb.0: ; %entry
4818 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4819 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4820 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4821 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4822 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4823 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4824 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4825 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4826 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4827 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4828 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4829 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4830 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4831 ; GFX7-NEXT: s_endpgm
4832 ;
4833 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4834 ; GFX10-WGP: ; %bb.0: ; %entry
4835 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4836 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4837 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4838 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4839 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4840 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4841 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4842 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4843 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4844 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4845 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4846 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4847 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4848 ; GFX10-WGP-NEXT: s_endpgm
4849 ;
4850 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4851 ; GFX10-CU: ; %bb.0: ; %entry
4852 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4853 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4854 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4855 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4856 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
4857 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
4858 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
4859 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
4860 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4861 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
4862 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
4863 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4864 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
4865 ; GFX10-CU-NEXT: s_endpgm
4866 ;
4867 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4868 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4869 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4870 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4871 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
4872 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
4873 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
4874 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
4875 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
4876 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
4877 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4878 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
4879 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
4880 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4881 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
4882 ; SKIP-CACHE-INV-NEXT: s_endpgm
4883 ;
4884 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4885 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4886 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4887 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4888 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4889 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4890 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4891 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4892 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
4893 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4894 ;
4895 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4896 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4897 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4898 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4899 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
4900 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
4901 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
4902 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4903 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
4904 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4905 ;
4906 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4907 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4908 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4909 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4910 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4911 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4912 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4913 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4914 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4915 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4916 ;
4917 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4918 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4919 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
4920 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4921 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
4922 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
4923 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
4924 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4925 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
4926 ; GFX940-TGSPLIT-NEXT: s_endpgm
4927 ;
4928 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4929 ; GFX11-WGP: ; %bb.0: ; %entry
4930 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4931 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4932 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4933 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4934 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4935 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4936 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
4937 ; GFX11-WGP-NEXT: s_endpgm
4938 ;
4939 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
4940 ; GFX11-CU: ; %bb.0: ; %entry
4941 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
4942 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4943 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
4944 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
4945 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
4946 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4947 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
4948 ; GFX11-CU-NEXT: s_endpgm
4949 ptr %out, i32 %in, i32 %old) {
4950 entry:
4951 %gep = getelementptr i32, ptr %out, i32 4
4952 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
4953 %val0 = extractvalue { i32, i1 } %val, 0
4954 store i32 %val0, ptr %out, align 4
4955 ret void
4956 }
4958 define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
4959 ; GFX7-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
4960 ; GFX7: ; %bb.0: ; %entry
4961 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4962 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4963 ; GFX7-NEXT: s_add_u32 s4, s0, 16
4964 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
4965 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4966 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
4967 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4968 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
4969 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4970 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
4971 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
4972 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4973 ; GFX7-NEXT: flat_store_dword v[0:1], v2
4974 ; GFX7-NEXT: s_endpgm
4975 ;
4976 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
4977 ; GFX10-WGP: ; %bb.0: ; %entry
4978 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4979 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4980 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
4981 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
4982 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
4983 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
4984 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
4985 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
4986 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4987 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
4988 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
4989 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4990 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
4991 ; GFX10-WGP-NEXT: s_endpgm
4992 ;
4993 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
4994 ; GFX10-CU: ; %bb.0: ; %entry
4995 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
4996 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4997 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
4998 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
4999 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5000 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5001 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5002 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5003 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5004 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5005 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5006 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5007 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5008 ; GFX10-CU-NEXT: s_endpgm
5009 ;
5010 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5011 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5012 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5013 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5014 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5015 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5016 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5017 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5018 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5019 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5020 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5021 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5022 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5023 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5024 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5025 ; SKIP-CACHE-INV-NEXT: s_endpgm
5026 ;
5027 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5028 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5029 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5030 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5031 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5032 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5033 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5034 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5035 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5036 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5037 ;
5038 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5039 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5040 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5041 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5042 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5043 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5044 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5045 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5046 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5047 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5048 ;
5049 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5050 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5051 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5052 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5053 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5054 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5055 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5056 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5057 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5058 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5059 ;
5060 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5061 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5062 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5063 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5064 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5065 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5066 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5067 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5068 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5069 ; GFX940-TGSPLIT-NEXT: s_endpgm
5070 ;
5071 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5072 ; GFX11-WGP: ; %bb.0: ; %entry
5073 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5074 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5075 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5076 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5077 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5078 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5079 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5080 ; GFX11-WGP-NEXT: s_endpgm
5081 ;
5082 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
5083 ; GFX11-CU: ; %bb.0: ; %entry
5084 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5085 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5086 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5087 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5088 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5089 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5090 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5091 ; GFX11-CU-NEXT: s_endpgm
5092 ptr %out, i32 %in, i32 %old) {
5093 entry:
5094 %gep = getelementptr i32, ptr %out, i32 4
5095 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
5096 %val0 = extractvalue { i32, i1 } %val, 0
5097 store i32 %val0, ptr %out, align 4
5098 ret void
5099 }
5101 define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
5102 ; GFX7-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5103 ; GFX7: ; %bb.0: ; %entry
5104 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5105 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5106 ; GFX7-NEXT: s_add_u32 s4, s0, 16
5107 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
5108 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5109 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
5110 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5111 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
5112 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5113 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5114 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5115 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5116 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5117 ; GFX7-NEXT: s_endpgm
5118 ;
5119 ; GFX10-WGP-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5120 ; GFX10-WGP: ; %bb.0: ; %entry
5121 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5122 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5123 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
5124 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
5125 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5126 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
5127 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5128 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
5129 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5130 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5131 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5132 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5133 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5134 ; GFX10-WGP-NEXT: s_endpgm
5135 ;
5136 ; GFX10-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5137 ; GFX10-CU: ; %bb.0: ; %entry
5138 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5139 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5140 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
5141 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
5142 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5143 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5144 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5145 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5146 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5147 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5148 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5149 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5150 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5151 ; GFX10-CU-NEXT: s_endpgm
5152 ;
5153 ; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5154 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5155 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5156 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5157 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5158 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5159 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5160 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5161 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5162 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5163 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5164 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5165 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5166 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5167 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5168 ; SKIP-CACHE-INV-NEXT: s_endpgm
5169 ;
5170 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5171 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5172 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5173 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5174 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5175 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5176 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5177 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5178 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5179 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5180 ;
5181 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5182 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5183 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5184 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5185 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5186 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5187 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5188 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5189 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5190 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5191 ;
5192 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5193 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5194 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5195 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5196 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5197 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5198 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5199 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5200 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5201 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5202 ;
5203 ; GFX940-TGSPLIT-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5204 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5205 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5206 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5207 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5208 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5209 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5210 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5211 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5212 ; GFX940-TGSPLIT-NEXT: s_endpgm
5213 ;
5214 ; GFX11-WGP-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5215 ; GFX11-WGP: ; %bb.0: ; %entry
5216 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5217 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5218 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5219 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5220 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5221 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5222 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5223 ; GFX11-WGP-NEXT: s_endpgm
5224 ;
5225 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
5226 ; GFX11-CU: ; %bb.0: ; %entry
5227 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5228 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5229 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5230 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5231 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5232 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5233 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5234 ; GFX11-CU-NEXT: s_endpgm
5235 ptr %out, i32 %in, i32 %old) {
5236 entry:
5237 %gep = getelementptr i32, ptr %out, i32 4
5238 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") monotonic seq_cst
5239 %val0 = extractvalue { i32, i1 } %val, 0
5240 store i32 %val0, ptr %out, align 4
5241 ret void
5242 }
5244 define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
5245 ; GFX7-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5246 ; GFX7: ; %bb.0: ; %entry
5247 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5248 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5249 ; GFX7-NEXT: s_add_u32 s4, s0, 16
5250 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
5251 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5252 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
5253 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5254 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
5255 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5256 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5257 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5258 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5259 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5260 ; GFX7-NEXT: s_endpgm
5261 ;
5262 ; GFX10-WGP-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5263 ; GFX10-WGP: ; %bb.0: ; %entry
5264 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5265 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5266 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
5267 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
5268 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
5270 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5271 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
5272 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5273 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5274 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5275 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5276 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5277 ; GFX10-WGP-NEXT: s_endpgm
5278 ;
5279 ; GFX10-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5280 ; GFX10-CU: ; %bb.0: ; %entry
5281 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5282 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5283 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
5284 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
5285 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5286 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5287 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5288 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5289 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5290 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5291 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5292 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5293 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5294 ; GFX10-CU-NEXT: s_endpgm
5295 ;
5296 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5297 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5298 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5299 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5300 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5301 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5302 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5304 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5305 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5306 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5308 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5309 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5310 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5311 ; SKIP-CACHE-INV-NEXT: s_endpgm
5312 ;
5313 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5314 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5315 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5316 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5317 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5318 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5319 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5320 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5321 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5322 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5323 ;
5324 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5325 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5326 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5327 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5328 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5329 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5330 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5331 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5332 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5333 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5334 ;
5335 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5336 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5337 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5338 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5339 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5340 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5341 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5342 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5343 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5344 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5345 ;
5346 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5347 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5348 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5349 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5350 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5351 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5352 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5353 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5354 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5355 ; GFX940-TGSPLIT-NEXT: s_endpgm
5356 ;
5357 ; GFX11-WGP-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5358 ; GFX11-WGP: ; %bb.0: ; %entry
5359 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5360 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5361 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5362 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5363 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5364 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5365 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5366 ; GFX11-WGP-NEXT: s_endpgm
5367 ;
5368 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
5369 ; GFX11-CU: ; %bb.0: ; %entry
5370 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5371 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5372 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5373 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5374 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5375 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5376 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5377 ; GFX11-CU-NEXT: s_endpgm
5378 ptr %out, i32 %in, i32 %old) {
5379 entry:
5380 %gep = getelementptr i32, ptr %out, i32 4
5381 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acquire seq_cst
5382 %val0 = extractvalue { i32, i1 } %val, 0
5383 store i32 %val0, ptr %out, align 4
5384 ret void
5385 }
5387 define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
5388 ; GFX7-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5389 ; GFX7: ; %bb.0: ; %entry
5390 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5391 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5392 ; GFX7-NEXT: s_add_u32 s4, s0, 16
5393 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
5394 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5395 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
5396 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5397 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
5398 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5399 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5400 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5401 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5402 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5403 ; GFX7-NEXT: s_endpgm
5404 ;
5405 ; GFX10-WGP-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5406 ; GFX10-WGP: ; %bb.0: ; %entry
5407 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5408 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5409 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
5410 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
5411 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5412 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
5413 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5414 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
5415 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5416 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5417 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5418 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5419 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5420 ; GFX10-WGP-NEXT: s_endpgm
5421 ;
5422 ; GFX10-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5423 ; GFX10-CU: ; %bb.0: ; %entry
5424 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5425 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5426 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
5427 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
5428 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5429 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5430 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5431 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5432 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5433 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5434 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5435 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5436 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5437 ; GFX10-CU-NEXT: s_endpgm
5438 ;
5439 ; SKIP-CACHE-INV-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5440 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5441 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5442 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5443 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5444 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5445 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5446 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5447 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5448 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5449 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5450 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5451 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5452 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5453 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5454 ; SKIP-CACHE-INV-NEXT: s_endpgm
5455 ;
5456 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5457 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5458 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5459 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5460 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5461 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5462 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5463 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5464 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5465 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5466 ;
5467 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5468 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5469 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5470 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5471 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5472 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5473 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5474 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5475 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5476 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5477 ;
5478 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5479 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5480 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5481 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5482 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5483 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5484 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5485 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5486 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5487 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5488 ;
5489 ; GFX940-TGSPLIT-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5490 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5491 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5492 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5493 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5494 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5495 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5496 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5497 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5498 ; GFX940-TGSPLIT-NEXT: s_endpgm
5499 ;
5500 ; GFX11-WGP-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5501 ; GFX11-WGP: ; %bb.0: ; %entry
5502 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5503 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5504 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5505 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5506 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5507 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5508 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5509 ; GFX11-WGP-NEXT: s_endpgm
5510 ;
5511 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
5512 ; GFX11-CU: ; %bb.0: ; %entry
5513 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5514 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5515 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5516 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5517 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5518 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5519 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5520 ; GFX11-CU-NEXT: s_endpgm
5521 ptr %out, i32 %in, i32 %old) {
5522 entry:
5523 %gep = getelementptr i32, ptr %out, i32 4
5524 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") release seq_cst
5525 %val0 = extractvalue { i32, i1 } %val, 0
5526 store i32 %val0, ptr %out, align 4
5527 ret void
5528 }
5530 define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
5531 ; GFX7-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5532 ; GFX7: ; %bb.0: ; %entry
5533 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5534 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5535 ; GFX7-NEXT: s_add_u32 s4, s0, 16
5536 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
5537 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5538 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
5539 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5540 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
5541 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5542 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5543 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5544 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5545 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5546 ; GFX7-NEXT: s_endpgm
5547 ;
5548 ; GFX10-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5549 ; GFX10-WGP: ; %bb.0: ; %entry
5550 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5551 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5552 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
5553 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
5554 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5555 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
5556 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5557 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
5558 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5559 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5560 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5561 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5562 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5563 ; GFX10-WGP-NEXT: s_endpgm
5564 ;
5565 ; GFX10-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5566 ; GFX10-CU: ; %bb.0: ; %entry
5567 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5568 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5569 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
5570 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
5571 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5572 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5573 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5574 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5575 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5576 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5577 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5578 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5579 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5580 ; GFX10-CU-NEXT: s_endpgm
5581 ;
5582 ; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5583 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5584 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5585 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5586 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5587 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5588 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5589 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5590 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5591 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5592 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5593 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5594 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5595 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5596 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5597 ; SKIP-CACHE-INV-NEXT: s_endpgm
5598 ;
5599 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5600 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5601 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5602 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5603 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5604 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5605 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5606 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5607 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5608 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5609 ;
5610 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5611 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5612 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5613 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5614 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5615 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5616 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5617 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5618 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5619 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5620 ;
5621 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5622 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5623 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5624 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5625 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5626 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5627 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5628 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5629 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5630 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5631 ;
5632 ; GFX940-TGSPLIT-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5633 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5634 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5635 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5636 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5637 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5638 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5639 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5640 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5641 ; GFX940-TGSPLIT-NEXT: s_endpgm
5642 ;
5643 ; GFX11-WGP-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5644 ; GFX11-WGP: ; %bb.0: ; %entry
5645 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5646 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5647 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5648 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5649 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5650 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5651 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5652 ; GFX11-WGP-NEXT: s_endpgm
5653 ;
5654 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
5655 ; GFX11-CU: ; %bb.0: ; %entry
5656 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5657 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5658 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5659 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5660 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5661 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5662 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5663 ; GFX11-CU-NEXT: s_endpgm
5664 ptr %out, i32 %in, i32 %old) {
5665 entry:
5666 %gep = getelementptr i32, ptr %out, i32 4
5667 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel seq_cst
5668 %val0 = extractvalue { i32, i1 } %val, 0
5669 store i32 %val0, ptr %out, align 4
5670 ret void
5671 }
5673 define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
5674 ; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5675 ; GFX7: ; %bb.0: ; %entry
5676 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5677 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5678 ; GFX7-NEXT: s_add_u32 s4, s0, 16
5679 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
5680 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5681 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
5682 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5683 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
5684 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5685 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5686 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5687 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5688 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5689 ; GFX7-NEXT: s_endpgm
5690 ;
5691 ; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5692 ; GFX10-WGP: ; %bb.0: ; %entry
5693 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5694 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5695 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
5696 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
5697 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
5698 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
5699 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
5700 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
5701 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5702 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5703 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5704 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5705 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5706 ; GFX10-WGP-NEXT: s_endpgm
5707 ;
5708 ; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5709 ; GFX10-CU: ; %bb.0: ; %entry
5710 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5711 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5712 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
5713 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
5714 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
5715 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
5716 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
5717 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
5718 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5719 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5720 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5721 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5722 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5723 ; GFX10-CU-NEXT: s_endpgm
5724 ;
5725 ; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5726 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5727 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5728 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5729 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
5730 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
5731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
5732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
5733 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
5734 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
5735 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
5736 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5737 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5738 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5739 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5740 ; SKIP-CACHE-INV-NEXT: s_endpgm
5741 ;
5742 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5743 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5744 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5745 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5746 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5747 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5748 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5749 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5750 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5751 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5752 ;
5753 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5754 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5755 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5756 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5757 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
5758 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
5759 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
5760 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5761 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5762 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5763 ;
5764 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5765 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5766 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5767 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5768 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5769 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5770 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5771 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5772 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5773 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5774 ;
5775 ; GFX940-TGSPLIT-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5776 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5777 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5778 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5779 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
5780 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
5781 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
5782 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5783 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5784 ; GFX940-TGSPLIT-NEXT: s_endpgm
5785 ;
5786 ; GFX11-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5787 ; GFX11-WGP: ; %bb.0: ; %entry
5788 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5789 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5790 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5791 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5792 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5793 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5794 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5795 ; GFX11-WGP-NEXT: s_endpgm
5796 ;
5797 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
5798 ; GFX11-CU: ; %bb.0: ; %entry
5799 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5800 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5801 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5802 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
5803 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
5804 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5805 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5806 ; GFX11-CU-NEXT: s_endpgm
5807 ptr %out, i32 %in, i32 %old) {
5808 entry:
5809 %gep = getelementptr i32, ptr %out, i32 4
5810 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
5811 %val0 = extractvalue { i32, i1 } %val, 0
5812 store i32 %val0, ptr %out, align 4
5813 ret void
5814 }
5816 define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
5817 ; GFX7-LABEL: flat_wavefront_one_as_unordered_load:
5818 ; GFX7: ; %bb.0: ; %entry
5819 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5820 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5821 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5822 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5823 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
5824 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
5825 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
5826 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5827 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5828 ; GFX7-NEXT: s_endpgm
5829 ;
5830 ; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_load:
5831 ; GFX10-WGP: ; %bb.0: ; %entry
5832 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5833 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5834 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5835 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5836 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
5837 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
5838 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
5839 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5840 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5841 ; GFX10-WGP-NEXT: s_endpgm
5842 ;
5843 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_load:
5844 ; GFX10-CU: ; %bb.0: ; %entry
5845 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5846 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5847 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5848 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5849 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
5850 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
5851 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
5852 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5853 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5854 ; GFX10-CU-NEXT: s_endpgm
5855 ;
5856 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_unordered_load:
5857 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5858 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5859 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5860 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5861 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5862 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
5863 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5864 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
5865 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5866 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
5867 ; SKIP-CACHE-INV-NEXT: s_endpgm
5868 ;
5869 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
5870 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5871 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5872 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5873 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
5874 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
5875 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
5876 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5877 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
5878 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5879 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
5880 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5881 ;
5882 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
5883 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5884 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5885 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5886 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
5887 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
5888 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
5889 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5890 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
5891 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5892 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
5893 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5894 ;
5895 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
5896 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5897 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5898 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5899 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
5900 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
5901 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
5902 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5903 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
5904 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5905 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5906 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5907 ;
5908 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_load:
5909 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5910 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5911 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5912 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
5913 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
5914 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
5915 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
5916 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
5917 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5918 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
5919 ; GFX940-TGSPLIT-NEXT: s_endpgm
5920 ;
5921 ; GFX11-WGP-LABEL: flat_wavefront_one_as_unordered_load:
5922 ; GFX11-WGP: ; %bb.0: ; %entry
5923 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5924 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5925 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5926 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
5927 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5928 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5929 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
5930 ; GFX11-WGP-NEXT: s_endpgm
5931 ;
5932 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_load:
5933 ; GFX11-CU: ; %bb.0: ; %entry
5934 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
5935 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5936 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
5937 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
5938 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
5939 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5940 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
5941 ; GFX11-CU-NEXT: s_endpgm
5942 ptr %in, ptr %out) {
5943 entry:
5944 %val = load atomic i32, ptr %in syncscope("wavefront-one-as") unordered, align 4
5945 store i32 %val, ptr %out
5946 ret void
5947 }
5949 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
5950 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_load:
5951 ; GFX7: ; %bb.0: ; %entry
5952 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5953 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5954 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
5955 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
5956 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
5957 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
5958 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
5959 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5960 ; GFX7-NEXT: flat_store_dword v[0:1], v2
5961 ; GFX7-NEXT: s_endpgm
5962 ;
5963 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_load:
5964 ; GFX10-WGP: ; %bb.0: ; %entry
5965 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5966 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5967 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
5968 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
5969 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
5970 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
5971 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
5972 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5973 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
5974 ; GFX10-WGP-NEXT: s_endpgm
5975 ;
5976 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_load:
5977 ; GFX10-CU: ; %bb.0: ; %entry
5978 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
5979 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5980 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
5981 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
5982 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
5983 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
5984 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
5985 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5986 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
5987 ; GFX10-CU-NEXT: s_endpgm
5988 ;
5989 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_load:
5990 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5991 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
5992 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5993 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
5994 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
5995 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
5996 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
5997 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
5998 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5999 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6000 ; SKIP-CACHE-INV-NEXT: s_endpgm
6001 ;
6002 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
6003 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6004 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6005 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6006 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6007 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6008 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6009 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6010 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6011 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6012 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6013 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6014 ;
6015 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
6016 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6017 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6018 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6019 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6020 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6021 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6022 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6023 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6024 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6025 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6026 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6027 ;
6028 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
6029 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6030 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6031 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6032 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6033 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6034 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6035 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6036 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6037 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6038 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6039 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6040 ;
6041 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_load:
6042 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6043 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6044 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6045 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6046 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6047 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6048 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6049 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6050 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6051 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6052 ; GFX940-TGSPLIT-NEXT: s_endpgm
6053 ;
6054 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_load:
6055 ; GFX11-WGP: ; %bb.0: ; %entry
6056 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6057 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6058 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6059 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
6060 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6061 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6062 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6063 ; GFX11-WGP-NEXT: s_endpgm
6064 ;
6065 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_load:
6066 ; GFX11-CU: ; %bb.0: ; %entry
6067 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6068 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6069 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6070 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
6071 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6072 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6073 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6074 ; GFX11-CU-NEXT: s_endpgm
6075 ptr %in, ptr %out) {
6076 entry:
6077 %val = load atomic i32, ptr %in syncscope("wavefront-one-as") monotonic, align 4
6078 store i32 %val, ptr %out
6079 ret void
6080 }
6082 define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
6083 ; GFX7-LABEL: flat_wavefront_one_as_acquire_load:
6084 ; GFX7: ; %bb.0: ; %entry
6085 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6086 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6087 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6088 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6089 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
6090 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
6091 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
6092 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6093 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6094 ; GFX7-NEXT: s_endpgm
6095 ;
6096 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_load:
6097 ; GFX10-WGP: ; %bb.0: ; %entry
6098 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6099 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6100 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6101 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6102 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
6103 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
6104 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
6105 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6106 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6107 ; GFX10-WGP-NEXT: s_endpgm
6108 ;
6109 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_load:
6110 ; GFX10-CU: ; %bb.0: ; %entry
6111 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6112 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6113 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6114 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6115 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
6116 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
6117 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
6118 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6119 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6120 ; GFX10-CU-NEXT: s_endpgm
6121 ;
6122 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_load:
6123 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6124 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6125 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6126 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6127 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6128 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
6129 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6130 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6131 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6132 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6133 ; SKIP-CACHE-INV-NEXT: s_endpgm
6134 ;
6135 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
6136 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6137 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6138 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6139 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6140 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6141 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6142 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6143 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6144 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6145 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6146 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6147 ;
6148 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
6149 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6150 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6151 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6152 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6153 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6154 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6155 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6156 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6157 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6158 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6159 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6160 ;
6161 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
6162 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6163 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6164 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6165 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6166 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6167 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6168 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6169 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6170 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6171 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6172 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6173 ;
6174 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_load:
6175 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6176 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6177 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6178 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6179 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6180 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6181 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6182 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6183 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6184 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6185 ; GFX940-TGSPLIT-NEXT: s_endpgm
6186 ;
6187 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_load:
6188 ; GFX11-WGP: ; %bb.0: ; %entry
6189 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6190 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6191 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6192 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
6193 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6194 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6195 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6196 ; GFX11-WGP-NEXT: s_endpgm
6197 ;
6198 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_load:
6199 ; GFX11-CU: ; %bb.0: ; %entry
6200 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6201 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6202 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6203 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
6204 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6205 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6206 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6207 ; GFX11-CU-NEXT: s_endpgm
6208 ptr %in, ptr %out) {
6209 entry:
6210 %val = load atomic i32, ptr %in syncscope("wavefront-one-as") acquire, align 4
6211 store i32 %val, ptr %out
6212 ret void
6213 }
6215 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
6216 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_load:
6217 ; GFX7: ; %bb.0: ; %entry
6218 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6219 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6220 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6221 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6222 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
6223 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
6224 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
6225 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6226 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6227 ; GFX7-NEXT: s_endpgm
6228 ;
6229 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_load:
6230 ; GFX10-WGP: ; %bb.0: ; %entry
6231 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6232 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6233 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6234 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6235 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
6236 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
6237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
6238 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6239 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6240 ; GFX10-WGP-NEXT: s_endpgm
6241 ;
6242 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
6243 ; GFX10-CU: ; %bb.0: ; %entry
6244 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6245 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6246 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6247 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6248 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
6249 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
6250 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
6251 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6252 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6253 ; GFX10-CU-NEXT: s_endpgm
6254 ;
6255 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_load:
6256 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6257 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6258 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6259 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
6260 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
6261 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
6262 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6263 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6264 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6265 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6266 ; SKIP-CACHE-INV-NEXT: s_endpgm
6267 ;
6268 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
6269 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6270 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6271 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6272 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6273 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6274 ; GFX90A-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6275 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6276 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6277 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6278 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6279 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6280 ;
6281 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
6282 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6283 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
6284 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6285 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6286 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6287 ; GFX90A-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6288 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6289 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6290 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6291 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6292 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6293 ;
6294 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
6295 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6296 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6297 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6298 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6299 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6300 ; GFX940-NOTTGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6301 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6302 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6303 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6304 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6305 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6306 ;
6307 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_load:
6308 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6309 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
6310 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6311 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
6312 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s1
6313 ; GFX940-TGSPLIT-NEXT: flat_load_dword v2, v[0:1]
6314 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s2
6315 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s3
6316 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6317 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6318 ; GFX940-TGSPLIT-NEXT: s_endpgm
6319 ;
6320 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_load:
6321 ; GFX11-WGP: ; %bb.0: ; %entry
6322 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6323 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6324 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6325 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
6326 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6327 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6328 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6329 ; GFX11-WGP-NEXT: s_endpgm
6330 ;
6331 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
6332 ; GFX11-CU: ; %bb.0: ; %entry
6333 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
6334 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6335 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
6336 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
6337 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6338 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6339 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6340 ; GFX11-CU-NEXT: s_endpgm
6341 ptr %in, ptr %out) {
6342 entry:
6343 %val = load atomic i32, ptr %in syncscope("wavefront-one-as") seq_cst, align 4
6344 store i32 %val, ptr %out
6345 ret void
6346 }
6348 define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
6349 ; GFX7-LABEL: flat_wavefront_one_as_unordered_store:
6350 ; GFX7: ; %bb.0: ; %entry
6351 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
6352 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
6353 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6354 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6355 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6356 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6357 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6358 ; GFX7-NEXT: s_endpgm
6359 ;
6360 ; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_store:
6361 ; GFX10-WGP: ; %bb.0: ; %entry
6362 ; GFX10-WGP-NEXT: s_clause 0x1
6363 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6364 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
6365 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6366 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6367 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6368 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6369 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6370 ; GFX10-WGP-NEXT: s_endpgm
6371 ;
6372 ; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_store:
6373 ; GFX10-CU: ; %bb.0: ; %entry
6374 ; GFX10-CU-NEXT: s_clause 0x1
6375 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6376 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
6377 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6378 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6379 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6380 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6381 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6382 ; GFX10-CU-NEXT: s_endpgm
6383 ;
6384 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_unordered_store:
6385 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6386 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
6387 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
6388 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6389 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6390 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6392 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6393 ; SKIP-CACHE-INV-NEXT: s_endpgm
6394 ;
6395 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
6396 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6397 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6398 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6399 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6400 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6401 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6402 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6403 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6404 ;
6405 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
6406 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6407 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6408 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6409 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6410 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6411 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6412 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6413 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6414 ;
6415 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
6416 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6417 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6418 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6419 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6420 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6421 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6422 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6423 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6424 ;
6425 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_unordered_store:
6426 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6427 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6428 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6429 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6430 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6431 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6432 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6433 ; GFX940-TGSPLIT-NEXT: s_endpgm
6434 ;
6435 ; GFX11-WGP-LABEL: flat_wavefront_one_as_unordered_store:
6436 ; GFX11-WGP: ; %bb.0: ; %entry
6437 ; GFX11-WGP-NEXT: s_clause 0x1
6438 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6439 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
6440 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6441 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6442 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
6443 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6444 ; GFX11-WGP-NEXT: s_endpgm
6445 ;
6446 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_store:
6447 ; GFX11-CU: ; %bb.0: ; %entry
6448 ; GFX11-CU-NEXT: s_clause 0x1
6449 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6450 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
6451 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6452 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6453 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
6454 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6455 ; GFX11-CU-NEXT: s_endpgm
6456 i32 %in, ptr %out) {
6457 entry:
6458 store atomic i32 %in, ptr %out syncscope("wavefront-one-as") unordered, align 4
6459 ret void
6460 }
6462 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
6463 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_store:
6464 ; GFX7: ; %bb.0: ; %entry
6465 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
6466 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
6467 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6468 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6469 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6470 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6471 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6472 ; GFX7-NEXT: s_endpgm
6473 ;
6474 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_store:
6475 ; GFX10-WGP: ; %bb.0: ; %entry
6476 ; GFX10-WGP-NEXT: s_clause 0x1
6477 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6478 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
6479 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6480 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6481 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6482 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6483 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6484 ; GFX10-WGP-NEXT: s_endpgm
6485 ;
6486 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_store:
6487 ; GFX10-CU: ; %bb.0: ; %entry
6488 ; GFX10-CU-NEXT: s_clause 0x1
6489 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6490 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
6491 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6492 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6493 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6494 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6495 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6496 ; GFX10-CU-NEXT: s_endpgm
6497 ;
6498 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_store:
6499 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6500 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
6501 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
6502 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6503 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6504 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6505 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6506 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6507 ; SKIP-CACHE-INV-NEXT: s_endpgm
6508 ;
6509 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
6510 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6511 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6512 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6513 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6514 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6515 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6516 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6517 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6518 ;
6519 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
6520 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6521 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6522 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6523 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6524 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6525 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6526 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6527 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6528 ;
6529 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
6530 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6531 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6532 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6533 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6534 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6535 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6536 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6537 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6538 ;
6539 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_store:
6540 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6541 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6542 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6543 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6544 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6545 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6546 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6547 ; GFX940-TGSPLIT-NEXT: s_endpgm
6548 ;
6549 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_store:
6550 ; GFX11-WGP: ; %bb.0: ; %entry
6551 ; GFX11-WGP-NEXT: s_clause 0x1
6552 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6553 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
6554 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6555 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6556 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
6557 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6558 ; GFX11-WGP-NEXT: s_endpgm
6559 ;
6560 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_store:
6561 ; GFX11-CU: ; %bb.0: ; %entry
6562 ; GFX11-CU-NEXT: s_clause 0x1
6563 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6564 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
6565 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6566 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6567 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
6568 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6569 ; GFX11-CU-NEXT: s_endpgm
6570 i32 %in, ptr %out) {
6571 entry:
6572 store atomic i32 %in, ptr %out syncscope("wavefront-one-as") monotonic, align 4
6573 ret void
6574 }
6576 define amdgpu_kernel void @flat_wavefront_one_as_release_store(
6577 ; GFX7-LABEL: flat_wavefront_one_as_release_store:
6578 ; GFX7: ; %bb.0: ; %entry
6579 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
6580 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
6581 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6582 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6583 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6584 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6585 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6586 ; GFX7-NEXT: s_endpgm
6587 ;
6588 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_store:
6589 ; GFX10-WGP: ; %bb.0: ; %entry
6590 ; GFX10-WGP-NEXT: s_clause 0x1
6591 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6592 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
6593 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6594 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6595 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6596 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6597 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6598 ; GFX10-WGP-NEXT: s_endpgm
6599 ;
6600 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_store:
6601 ; GFX10-CU: ; %bb.0: ; %entry
6602 ; GFX10-CU-NEXT: s_clause 0x1
6603 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6604 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
6605 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6606 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6607 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6608 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6609 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6610 ; GFX10-CU-NEXT: s_endpgm
6611 ;
6612 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_store:
6613 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6614 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
6615 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
6616 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6617 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6618 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6619 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6620 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6621 ; SKIP-CACHE-INV-NEXT: s_endpgm
6622 ;
6623 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_store:
6624 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6625 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6626 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6627 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6628 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6629 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6630 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6631 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6632 ;
6633 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_store:
6634 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6635 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6636 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6637 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6638 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6639 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6640 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6641 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6642 ;
6643 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_store:
6644 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6645 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6646 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6647 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6648 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6649 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6650 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6651 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6652 ;
6653 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_store:
6654 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6655 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6656 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6657 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6658 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6659 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6660 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6661 ; GFX940-TGSPLIT-NEXT: s_endpgm
6662 ;
6663 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_store:
6664 ; GFX11-WGP: ; %bb.0: ; %entry
6665 ; GFX11-WGP-NEXT: s_clause 0x1
6666 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6667 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
6668 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6669 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6670 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
6671 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6672 ; GFX11-WGP-NEXT: s_endpgm
6673 ;
6674 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_store:
6675 ; GFX11-CU: ; %bb.0: ; %entry
6676 ; GFX11-CU-NEXT: s_clause 0x1
6677 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6678 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
6679 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6680 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6681 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
6682 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6683 ; GFX11-CU-NEXT: s_endpgm
6684 i32 %in, ptr %out) {
6685 entry:
6686 store atomic i32 %in, ptr %out syncscope("wavefront-one-as") release, align 4
6687 ret void
6688 }
6690 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
6691 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_store:
6692 ; GFX7: ; %bb.0: ; %entry
6693 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
6694 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
6695 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6696 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6697 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6698 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6699 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6700 ; GFX7-NEXT: s_endpgm
6701 ;
6702 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_store:
6703 ; GFX10-WGP: ; %bb.0: ; %entry
6704 ; GFX10-WGP-NEXT: s_clause 0x1
6705 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6706 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
6707 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6708 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6709 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6710 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6711 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
6712 ; GFX10-WGP-NEXT: s_endpgm
6713 ;
6714 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
6715 ; GFX10-CU: ; %bb.0: ; %entry
6716 ; GFX10-CU-NEXT: s_clause 0x1
6717 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6718 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
6719 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6720 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6721 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6722 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6723 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
6724 ; GFX10-CU-NEXT: s_endpgm
6725 ;
6726 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_store:
6727 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6728 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
6729 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
6730 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6733 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6734 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
6735 ; SKIP-CACHE-INV-NEXT: s_endpgm
6736 ;
6737 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
6738 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6739 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6740 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6741 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6742 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6743 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6744 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
6745 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6746 ;
6747 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
6748 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6749 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
6750 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
6751 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6752 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6753 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6754 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
6755 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6756 ;
6757 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
6758 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6759 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6760 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6761 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6762 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6763 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6764 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6765 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6766 ;
6767 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_store:
6768 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6769 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x8
6770 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x0
6771 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6772 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6773 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6774 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
6775 ; GFX940-TGSPLIT-NEXT: s_endpgm
6776 ;
6777 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_store:
6778 ; GFX11-WGP: ; %bb.0: ; %entry
6779 ; GFX11-WGP-NEXT: s_clause 0x1
6780 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6781 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
6782 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6783 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6784 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
6785 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
6786 ; GFX11-WGP-NEXT: s_endpgm
6787 ;
6788 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
6789 ; GFX11-CU: ; %bb.0: ; %entry
6790 ; GFX11-CU-NEXT: s_clause 0x1
6791 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
6792 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
6793 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6794 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6795 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
6796 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
6797 ; GFX11-CU-NEXT: s_endpgm
6798 i32 %in, ptr %out) {
6799 entry:
6800 store atomic i32 %in, ptr %out syncscope("wavefront-one-as") seq_cst, align 4
6801 ret void
6802 }
6804 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
6805 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6806 ; GFX7: ; %bb.0: ; %entry
6807 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6808 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
6809 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6810 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6811 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6812 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6813 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
6814 ; GFX7-NEXT: s_endpgm
6815 ;
6816 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6817 ; GFX10-WGP: ; %bb.0: ; %entry
6818 ; GFX10-WGP-NEXT: s_clause 0x1
6819 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6820 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
6821 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6822 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6823 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6824 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6825 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
6826 ; GFX10-WGP-NEXT: s_endpgm
6827 ;
6828 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6829 ; GFX10-CU: ; %bb.0: ; %entry
6830 ; GFX10-CU-NEXT: s_clause 0x1
6831 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6832 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
6833 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6834 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6835 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6836 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6837 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
6838 ; GFX10-CU-NEXT: s_endpgm
6839 ;
6840 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6841 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6842 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6843 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
6844 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6845 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6846 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6847 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6848 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
6849 ; SKIP-CACHE-INV-NEXT: s_endpgm
6850 ;
6851 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6852 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6853 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6854 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
6855 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6856 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6857 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6858 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6859 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6860 ;
6861 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6862 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6863 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6864 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
6865 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6866 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6867 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6868 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6869 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6870 ;
6871 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6872 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6873 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6874 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
6875 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6876 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6877 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6878 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6879 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6880 ;
6881 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6882 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6883 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6884 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
6885 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6886 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6887 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6888 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6889 ; GFX940-TGSPLIT-NEXT: s_endpgm
6890 ;
6891 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6892 ; GFX11-WGP: ; %bb.0: ; %entry
6893 ; GFX11-WGP-NEXT: s_clause 0x1
6894 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
6895 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
6896 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6897 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6898 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
6899 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
6900 ; GFX11-WGP-NEXT: s_endpgm
6901 ;
6902 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
6903 ; GFX11-CU: ; %bb.0: ; %entry
6904 ; GFX11-CU-NEXT: s_clause 0x1
6905 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
6906 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
6907 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6908 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
6909 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
6910 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
6911 ; GFX11-CU-NEXT: s_endpgm
6912 ptr %out, i32 %in) {
6913 entry:
6914 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") monotonic
6915 ret void
6916 }
6918 define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
6919 ; GFX7-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6920 ; GFX7: ; %bb.0: ; %entry
6921 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6922 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
6923 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6924 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
6925 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
6926 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
6927 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
6928 ; GFX7-NEXT: s_endpgm
6929 ;
6930 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6931 ; GFX10-WGP: ; %bb.0: ; %entry
6932 ; GFX10-WGP-NEXT: s_clause 0x1
6933 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6934 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
6935 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6936 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
6937 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
6938 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
6939 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
6940 ; GFX10-WGP-NEXT: s_endpgm
6941 ;
6942 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6943 ; GFX10-CU: ; %bb.0: ; %entry
6944 ; GFX10-CU-NEXT: s_clause 0x1
6945 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6946 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
6947 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6948 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
6949 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
6950 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
6951 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
6952 ; GFX10-CU-NEXT: s_endpgm
6953 ;
6954 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6955 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6956 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6957 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
6958 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6959 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
6960 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
6961 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
6962 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
6963 ; SKIP-CACHE-INV-NEXT: s_endpgm
6964 ;
6965 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6966 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6967 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6968 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
6969 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6970 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6971 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6972 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6973 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6974 ;
6975 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6976 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6977 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6978 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
6979 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6980 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
6981 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
6982 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6983 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6984 ;
6985 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6986 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6987 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6988 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
6989 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6990 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
6991 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
6992 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
6993 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6994 ;
6995 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
6996 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6997 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
6998 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
6999 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7000 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7001 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7002 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7003 ; GFX940-TGSPLIT-NEXT: s_endpgm
7004 ;
7005 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
7006 ; GFX11-WGP: ; %bb.0: ; %entry
7007 ; GFX11-WGP-NEXT: s_clause 0x1
7008 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7009 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7010 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7011 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7012 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7013 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
7014 ; GFX11-WGP-NEXT: s_endpgm
7015 ;
7016 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
7017 ; GFX11-CU: ; %bb.0: ; %entry
7018 ; GFX11-CU-NEXT: s_clause 0x1
7019 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7020 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7021 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7022 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7023 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7024 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
7025 ; GFX11-CU-NEXT: s_endpgm
7026 ptr %out, i32 %in) {
7027 entry:
7028 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
7029 ret void
7030 }
7032 define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
7033 ; GFX7-LABEL: flat_wavefront_one_as_release_atomicrmw:
7034 ; GFX7: ; %bb.0: ; %entry
7035 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7036 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7037 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7038 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7039 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7040 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7041 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
7042 ; GFX7-NEXT: s_endpgm
7043 ;
7044 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_atomicrmw:
7045 ; GFX10-WGP: ; %bb.0: ; %entry
7046 ; GFX10-WGP-NEXT: s_clause 0x1
7047 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7048 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7049 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7050 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7051 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7052 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7053 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
7054 ; GFX10-WGP-NEXT: s_endpgm
7055 ;
7056 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
7057 ; GFX10-CU: ; %bb.0: ; %entry
7058 ; GFX10-CU-NEXT: s_clause 0x1
7059 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7060 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7061 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7062 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7063 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7064 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7065 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
7066 ; GFX10-CU-NEXT: s_endpgm
7067 ;
7068 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_atomicrmw:
7069 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7070 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7071 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7072 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7073 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7074 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7075 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7076 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
7077 ; SKIP-CACHE-INV-NEXT: s_endpgm
7078 ;
7079 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
7080 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7081 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7082 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7083 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7084 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7085 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7086 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7087 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7088 ;
7089 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
7090 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7091 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7092 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7093 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7094 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7095 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7096 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7097 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7098 ;
7099 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
7100 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7101 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7102 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7103 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7104 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7105 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7106 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7107 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7108 ;
7109 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_atomicrmw:
7110 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7111 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7112 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7113 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7114 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7115 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7116 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7117 ; GFX940-TGSPLIT-NEXT: s_endpgm
7118 ;
7119 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_atomicrmw:
7120 ; GFX11-WGP: ; %bb.0: ; %entry
7121 ; GFX11-WGP-NEXT: s_clause 0x1
7122 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7123 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7124 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7125 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7126 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7127 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
7128 ; GFX11-WGP-NEXT: s_endpgm
7129 ;
7130 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
7131 ; GFX11-CU: ; %bb.0: ; %entry
7132 ; GFX11-CU-NEXT: s_clause 0x1
7133 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7134 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7135 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7136 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7137 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7138 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
7139 ; GFX11-CU-NEXT: s_endpgm
7140 ptr %out, i32 %in) {
7141 entry:
7142 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") release
7143 ret void
7144 }
7146 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
7147 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7148 ; GFX7: ; %bb.0: ; %entry
7149 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7150 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7151 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7152 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7153 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7154 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7155 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
7156 ; GFX7-NEXT: s_endpgm
7157 ;
7158 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7159 ; GFX10-WGP: ; %bb.0: ; %entry
7160 ; GFX10-WGP-NEXT: s_clause 0x1
7161 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7162 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7163 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7164 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7165 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7166 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7167 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
7168 ; GFX10-WGP-NEXT: s_endpgm
7169 ;
7170 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7171 ; GFX10-CU: ; %bb.0: ; %entry
7172 ; GFX10-CU-NEXT: s_clause 0x1
7173 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7174 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7175 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7176 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7177 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7178 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7179 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
7180 ; GFX10-CU-NEXT: s_endpgm
7181 ;
7182 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7183 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7184 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7185 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7186 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7187 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7188 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7189 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7190 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
7191 ; SKIP-CACHE-INV-NEXT: s_endpgm
7192 ;
7193 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7194 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7195 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7196 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7197 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7198 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7199 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7200 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7201 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7202 ;
7203 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7204 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7205 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7206 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7207 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7208 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7209 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7210 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7211 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7212 ;
7213 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7214 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7215 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7216 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7217 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7218 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7219 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7220 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7221 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7222 ;
7223 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7224 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7225 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7226 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7227 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7228 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7229 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7230 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7231 ; GFX940-TGSPLIT-NEXT: s_endpgm
7232 ;
7233 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7234 ; GFX11-WGP: ; %bb.0: ; %entry
7235 ; GFX11-WGP-NEXT: s_clause 0x1
7236 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7237 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7238 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7239 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7240 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7241 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
7242 ; GFX11-WGP-NEXT: s_endpgm
7243 ;
7244 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
7245 ; GFX11-CU: ; %bb.0: ; %entry
7246 ; GFX11-CU-NEXT: s_clause 0x1
7247 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7248 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7249 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7250 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7251 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7252 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
7253 ; GFX11-CU-NEXT: s_endpgm
7254 ptr %out, i32 %in) {
7255 entry:
7256 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
7257 ret void
7258 }
7260 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
7261 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7262 ; GFX7: ; %bb.0: ; %entry
7263 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7264 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7265 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7266 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7267 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7268 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7269 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
7270 ; GFX7-NEXT: s_endpgm
7271 ;
7272 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7273 ; GFX10-WGP: ; %bb.0: ; %entry
7274 ; GFX10-WGP-NEXT: s_clause 0x1
7275 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7276 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7277 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7278 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7279 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7280 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7281 ; GFX10-WGP-NEXT: flat_atomic_swap v[0:1], v2
7282 ; GFX10-WGP-NEXT: s_endpgm
7283 ;
7284 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7285 ; GFX10-CU: ; %bb.0: ; %entry
7286 ; GFX10-CU-NEXT: s_clause 0x1
7287 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7288 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7289 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7290 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7291 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7292 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7293 ; GFX10-CU-NEXT: flat_atomic_swap v[0:1], v2
7294 ; GFX10-CU-NEXT: s_endpgm
7295 ;
7296 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7297 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7298 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7299 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7300 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7301 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7302 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7304 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v[0:1], v2
7305 ; SKIP-CACHE-INV-NEXT: s_endpgm
7306 ;
7307 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7308 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7309 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7310 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7311 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7312 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7313 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7314 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7315 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7316 ;
7317 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7318 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7319 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7320 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7321 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7322 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7323 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7324 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7325 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7326 ;
7327 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7328 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7329 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7330 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7331 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7332 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7333 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7334 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7335 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7336 ;
7337 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7338 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7339 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7340 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7341 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7342 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7343 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7344 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v[0:1], v2
7345 ; GFX940-TGSPLIT-NEXT: s_endpgm
7346 ;
7347 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7348 ; GFX11-WGP: ; %bb.0: ; %entry
7349 ; GFX11-WGP-NEXT: s_clause 0x1
7350 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7351 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7352 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7353 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7354 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7355 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v[0:1], v2
7356 ; GFX11-WGP-NEXT: s_endpgm
7357 ;
7358 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
7359 ; GFX11-CU: ; %bb.0: ; %entry
7360 ; GFX11-CU-NEXT: s_clause 0x1
7361 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7362 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7363 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7364 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7365 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7366 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v[0:1], v2
7367 ; GFX11-CU-NEXT: s_endpgm
7368 ptr %out, i32 %in) {
7369 entry:
7370 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
7371 ret void
7372 }
7374 define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
7375 ; GFX7-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7376 ; GFX7: ; %bb.0: ; %entry
7377 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7378 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7379 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7380 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7381 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7382 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7383 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7384 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7385 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7386 ; GFX7-NEXT: s_endpgm
7387 ;
7388 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7389 ; GFX10-WGP: ; %bb.0: ; %entry
7390 ; GFX10-WGP-NEXT: s_clause 0x1
7391 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7392 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7393 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7394 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7395 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7396 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7397 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7398 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7399 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7400 ; GFX10-WGP-NEXT: s_endpgm
7401 ;
7402 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7403 ; GFX10-CU: ; %bb.0: ; %entry
7404 ; GFX10-CU-NEXT: s_clause 0x1
7405 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7406 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7407 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7408 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7409 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7410 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7411 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7412 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7413 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7414 ; GFX10-CU-NEXT: s_endpgm
7415 ;
7416 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7417 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7418 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7419 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7420 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7421 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7422 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7423 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7424 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7425 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7426 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7427 ; SKIP-CACHE-INV-NEXT: s_endpgm
7428 ;
7429 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7430 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7431 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7432 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7433 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7434 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7435 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7436 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7437 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7438 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7439 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7440 ;
7441 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7442 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7443 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7444 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7445 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7446 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7447 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7448 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7449 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7450 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7451 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7452 ;
7453 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7454 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7455 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7456 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7457 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7458 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7459 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7460 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7461 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7462 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7463 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7464 ;
7465 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7466 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7467 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7468 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7469 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7470 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7471 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7472 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7473 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7474 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7475 ; GFX940-TGSPLIT-NEXT: s_endpgm
7476 ;
7477 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7478 ; GFX11-WGP: ; %bb.0: ; %entry
7479 ; GFX11-WGP-NEXT: s_clause 0x1
7480 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7481 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7482 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7483 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7484 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7485 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7486 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7487 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7488 ; GFX11-WGP-NEXT: s_endpgm
7489 ;
7490 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
7491 ; GFX11-CU: ; %bb.0: ; %entry
7492 ; GFX11-CU-NEXT: s_clause 0x1
7493 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7494 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7495 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7496 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7497 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7498 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7499 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7500 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7501 ; GFX11-CU-NEXT: s_endpgm
7502 ptr %out, i32 %in) {
7503 entry:
7504 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
7505 store i32 %val, ptr %out, align 4
7506 ret void
7507 }
7509 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
7510 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7511 ; GFX7: ; %bb.0: ; %entry
7512 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7513 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7514 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7515 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7516 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7517 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7518 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7519 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7520 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7521 ; GFX7-NEXT: s_endpgm
7522 ;
7523 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7524 ; GFX10-WGP: ; %bb.0: ; %entry
7525 ; GFX10-WGP-NEXT: s_clause 0x1
7526 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7527 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7528 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7529 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7530 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7531 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7532 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7533 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7534 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7535 ; GFX10-WGP-NEXT: s_endpgm
7536 ;
7537 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7538 ; GFX10-CU: ; %bb.0: ; %entry
7539 ; GFX10-CU-NEXT: s_clause 0x1
7540 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7541 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7542 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7543 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7544 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7545 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7546 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7547 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7548 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7549 ; GFX10-CU-NEXT: s_endpgm
7550 ;
7551 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7552 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7553 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7554 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7555 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7556 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7557 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7559 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7560 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7561 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7562 ; SKIP-CACHE-INV-NEXT: s_endpgm
7563 ;
7564 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7565 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7566 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7567 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7568 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7569 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7570 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7571 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7572 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7573 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7574 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7575 ;
7576 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7577 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7578 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7579 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7580 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7581 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7582 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7583 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7584 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7585 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7586 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7587 ;
7588 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7589 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7590 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7591 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7592 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7593 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7594 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7595 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7596 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7597 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7598 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7599 ;
7600 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7601 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7602 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7603 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7604 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7605 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7606 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7607 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7608 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7609 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7610 ; GFX940-TGSPLIT-NEXT: s_endpgm
7611 ;
7612 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7613 ; GFX11-WGP: ; %bb.0: ; %entry
7614 ; GFX11-WGP-NEXT: s_clause 0x1
7615 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7616 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7617 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7618 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7619 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7620 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7621 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7622 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7623 ; GFX11-WGP-NEXT: s_endpgm
7624 ;
7625 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
7626 ; GFX11-CU: ; %bb.0: ; %entry
7627 ; GFX11-CU-NEXT: s_clause 0x1
7628 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7629 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7630 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7631 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7632 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7633 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7634 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7635 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7636 ; GFX11-CU-NEXT: s_endpgm
7637 ptr %out, i32 %in) {
7638 entry:
7639 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
7640 store i32 %val, ptr %out, align 4
7641 ret void
7642 }
7644 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
7645 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7646 ; GFX7: ; %bb.0: ; %entry
7647 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7648 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
7649 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7650 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7651 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7652 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7653 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7654 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7655 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7656 ; GFX7-NEXT: s_endpgm
7657 ;
7658 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7659 ; GFX10-WGP: ; %bb.0: ; %entry
7660 ; GFX10-WGP-NEXT: s_clause 0x1
7661 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7662 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x8
7663 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7664 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7665 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7666 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7667 ; GFX10-WGP-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7668 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7669 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
7670 ; GFX10-WGP-NEXT: s_endpgm
7671 ;
7672 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7673 ; GFX10-CU: ; %bb.0: ; %entry
7674 ; GFX10-CU-NEXT: s_clause 0x1
7675 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7676 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x8
7677 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7678 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7679 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7680 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7681 ; GFX10-CU-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7682 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7683 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
7684 ; GFX10-CU-NEXT: s_endpgm
7685 ;
7686 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7687 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7688 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7689 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
7690 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7691 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
7692 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
7693 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
7694 ; SKIP-CACHE-INV-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7695 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7696 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
7697 ; SKIP-CACHE-INV-NEXT: s_endpgm
7698 ;
7699 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7700 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7701 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7702 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7703 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7704 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7705 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7706 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7707 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7708 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
7709 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7710 ;
7711 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7712 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7713 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7714 ; GFX90A-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
7715 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7716 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7717 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s2
7718 ; GFX90A-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
7719 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7720 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
7721 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7722 ;
7723 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7724 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7725 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7726 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7727 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7728 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7729 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7730 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7731 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7732 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7733 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7734 ;
7735 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7736 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7737 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
7738 ; GFX940-TGSPLIT-NEXT: s_load_dword s4, s[0:1], 0x8
7739 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7740 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
7741 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s4
7742 ; GFX940-TGSPLIT-NEXT: flat_atomic_swap v2, v[0:1], v2 sc0
7743 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7744 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
7745 ; GFX940-TGSPLIT-NEXT: s_endpgm
7746 ;
7747 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7748 ; GFX11-WGP: ; %bb.0: ; %entry
7749 ; GFX11-WGP-NEXT: s_clause 0x1
7750 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7751 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
7752 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7753 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7754 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
7755 ; GFX11-WGP-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7756 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7757 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
7758 ; GFX11-WGP-NEXT: s_endpgm
7759 ;
7760 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
7761 ; GFX11-CU: ; %bb.0: ; %entry
7762 ; GFX11-CU-NEXT: s_clause 0x1
7763 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
7764 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
7765 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7766 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
7767 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
7768 ; GFX11-CU-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc
7769 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7770 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
7771 ; GFX11-CU-NEXT: s_endpgm
7772 ptr %out, i32 %in) {
7773 entry:
7774 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
7775 store i32 %val, ptr %out, align 4
7776 ret void
7777 }
7779 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
7780 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7781 ; GFX7: ; %bb.0: ; %entry
7782 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7783 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7784 ; GFX7-NEXT: s_add_u32 s0, s0, 16
7785 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
7786 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7787 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7788 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7789 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
7790 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7791 ; GFX7-NEXT: s_endpgm
7792 ;
7793 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7794 ; GFX10-WGP: ; %bb.0: ; %entry
7795 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7796 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7797 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
7798 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
7799 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7800 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7801 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7802 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
7803 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7804 ; GFX10-WGP-NEXT: s_endpgm
7805 ;
7806 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7807 ; GFX10-CU: ; %bb.0: ; %entry
7808 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7809 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7810 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
7811 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
7812 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7813 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7814 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7815 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
7816 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7817 ; GFX10-CU-NEXT: s_endpgm
7818 ;
7819 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7820 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7821 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7822 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7823 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
7824 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
7825 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7826 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
7827 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7828 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
7829 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7830 ; SKIP-CACHE-INV-NEXT: s_endpgm
7831 ;
7832 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7833 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7834 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7835 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7836 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7837 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7838 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7839 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7840 ;
7841 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7842 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7843 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7844 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7845 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7846 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7847 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7848 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7849 ;
7850 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7851 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7852 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7853 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7854 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7855 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
7856 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7857 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7858 ;
7859 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7860 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7861 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7862 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7863 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7864 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
7865 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7866 ; GFX940-TGSPLIT-NEXT: s_endpgm
7867 ;
7868 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7869 ; GFX11-WGP: ; %bb.0: ; %entry
7870 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
7871 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7872 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7873 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
7874 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
7875 ; GFX11-WGP-NEXT: s_endpgm
7876 ;
7877 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
7878 ; GFX11-CU: ; %bb.0: ; %entry
7879 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
7880 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7881 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7882 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
7883 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
7884 ; GFX11-CU-NEXT: s_endpgm
7885 ptr %out, i32 %in, i32 %old) {
7886 entry:
7887 %gep = getelementptr i32, ptr %out, i32 4
7888 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
7889 ret void
7890 }
7892 define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
7893 ; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7894 ; GFX7: ; %bb.0: ; %entry
7895 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7896 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7897 ; GFX7-NEXT: s_add_u32 s0, s0, 16
7898 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
7899 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
7900 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
7901 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
7902 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
7903 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7904 ; GFX7-NEXT: s_endpgm
7905 ;
7906 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7907 ; GFX10-WGP: ; %bb.0: ; %entry
7908 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7909 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7910 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
7911 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
7912 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
7913 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
7914 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
7915 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
7916 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7917 ; GFX10-WGP-NEXT: s_endpgm
7918 ;
7919 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7920 ; GFX10-CU: ; %bb.0: ; %entry
7921 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7922 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7923 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
7924 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
7925 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
7926 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
7927 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
7928 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
7929 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7930 ; GFX10-CU-NEXT: s_endpgm
7931 ;
7932 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7933 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7934 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7935 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7936 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
7937 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
7938 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
7939 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
7940 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
7941 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
7942 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
7943 ; SKIP-CACHE-INV-NEXT: s_endpgm
7944 ;
7945 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7946 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7947 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7948 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7949 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7950 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7951 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7952 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7953 ;
7954 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7955 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7956 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
7957 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7958 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
7959 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
7960 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7961 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7962 ;
7963 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7964 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7965 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7966 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7967 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7968 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
7969 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7970 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7971 ;
7972 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7973 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7974 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
7975 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7976 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
7977 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
7978 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
7979 ; GFX940-TGSPLIT-NEXT: s_endpgm
7980 ;
7981 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7982 ; GFX11-WGP: ; %bb.0: ; %entry
7983 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
7984 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7985 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7986 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
7987 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
7988 ; GFX11-WGP-NEXT: s_endpgm
7989 ;
7990 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
7991 ; GFX11-CU: ; %bb.0: ; %entry
7992 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
7993 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7994 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
7995 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
7996 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
7997 ; GFX11-CU-NEXT: s_endpgm
7998 ptr %out, i32 %in, i32 %old) {
7999 entry:
8000 %gep = getelementptr i32, ptr %out, i32 4
8001 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
8002 ret void
8003 }
8005 define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
8006 ; GFX7-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8007 ; GFX7: ; %bb.0: ; %entry
8008 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8009 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8010 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8011 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8012 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8013 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8014 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8015 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8016 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8017 ; GFX7-NEXT: s_endpgm
8018 ;
8019 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8020 ; GFX10-WGP: ; %bb.0: ; %entry
8021 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8022 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8023 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8024 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8025 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8026 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8027 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8028 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8029 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8030 ; GFX10-WGP-NEXT: s_endpgm
8031 ;
8032 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8033 ; GFX10-CU: ; %bb.0: ; %entry
8034 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8035 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8036 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8037 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8038 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8039 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8040 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8041 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8042 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8043 ; GFX10-CU-NEXT: s_endpgm
8044 ;
8045 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8046 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8047 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8048 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8049 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8050 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8051 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8052 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8053 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8054 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8055 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8056 ; SKIP-CACHE-INV-NEXT: s_endpgm
8057 ;
8058 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8059 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8060 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8061 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8062 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8063 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8064 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8065 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8066 ;
8067 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8068 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8069 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8070 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8071 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8072 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8073 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8074 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8075 ;
8076 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8077 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8078 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8079 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8080 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8081 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8082 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8083 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8084 ;
8085 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8086 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8087 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8088 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8089 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8090 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8091 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8092 ; GFX940-TGSPLIT-NEXT: s_endpgm
8093 ;
8094 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8095 ; GFX11-WGP: ; %bb.0: ; %entry
8096 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8097 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8098 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8099 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8100 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8101 ; GFX11-WGP-NEXT: s_endpgm
8102 ;
8103 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
8104 ; GFX11-CU: ; %bb.0: ; %entry
8105 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8106 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8107 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8108 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8109 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8110 ; GFX11-CU-NEXT: s_endpgm
8111 ptr %out, i32 %in, i32 %old) {
8112 entry:
8113 %gep = getelementptr i32, ptr %out, i32 4
8114 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
8115 ret void
8116 }
8118 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
8119 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8120 ; GFX7: ; %bb.0: ; %entry
8121 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8122 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8123 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8124 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8125 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8126 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8127 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8128 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8129 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8130 ; GFX7-NEXT: s_endpgm
8131 ;
8132 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8133 ; GFX10-WGP: ; %bb.0: ; %entry
8134 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8135 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8136 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8137 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8138 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8139 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8140 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8141 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8142 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8143 ; GFX10-WGP-NEXT: s_endpgm
8144 ;
8145 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8146 ; GFX10-CU: ; %bb.0: ; %entry
8147 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8148 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8149 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8150 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8151 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8152 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8153 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8154 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8155 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8156 ; GFX10-CU-NEXT: s_endpgm
8157 ;
8158 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8159 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8160 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8161 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8162 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8163 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8164 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8165 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8166 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8167 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8168 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8169 ; SKIP-CACHE-INV-NEXT: s_endpgm
8170 ;
8171 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8172 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8173 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8174 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8175 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8176 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8177 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8178 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8179 ;
8180 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8181 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8182 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8183 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8184 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8185 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8186 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8187 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8188 ;
8189 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8190 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8191 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8192 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8193 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8194 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8195 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8196 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8197 ;
8198 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8199 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8200 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8201 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8202 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8203 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8204 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8205 ; GFX940-TGSPLIT-NEXT: s_endpgm
8206 ;
8207 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8208 ; GFX11-WGP: ; %bb.0: ; %entry
8209 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8210 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8211 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8212 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8213 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8214 ; GFX11-WGP-NEXT: s_endpgm
8215 ;
8216 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
8217 ; GFX11-CU: ; %bb.0: ; %entry
8218 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8219 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8220 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8221 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8222 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8223 ; GFX11-CU-NEXT: s_endpgm
8224 ptr %out, i32 %in, i32 %old) {
8225 entry:
8226 %gep = getelementptr i32, ptr %out, i32 4
8227 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
8228 ret void
8229 }
8231 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
8232 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8233 ; GFX7: ; %bb.0: ; %entry
8234 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8235 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8236 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8237 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8238 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8239 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8240 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8241 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8242 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8243 ; GFX7-NEXT: s_endpgm
8244 ;
8245 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8246 ; GFX10-WGP: ; %bb.0: ; %entry
8247 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8248 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8249 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8250 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8251 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8252 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8253 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8254 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8255 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8256 ; GFX10-WGP-NEXT: s_endpgm
8257 ;
8258 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8259 ; GFX10-CU: ; %bb.0: ; %entry
8260 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8261 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8262 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8263 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8264 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8265 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8266 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8267 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8268 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8269 ; GFX10-CU-NEXT: s_endpgm
8270 ;
8271 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8272 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8273 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8274 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8275 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8276 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8277 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8278 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8279 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8280 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8281 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8282 ; SKIP-CACHE-INV-NEXT: s_endpgm
8283 ;
8284 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8285 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8286 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8287 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8288 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8289 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8290 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8291 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8292 ;
8293 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8294 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8295 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8296 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8297 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8298 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8299 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8300 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8301 ;
8302 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8303 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8304 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8305 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8306 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8307 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8308 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8309 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8310 ;
8311 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8312 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8313 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8314 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8315 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8316 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8317 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8318 ; GFX940-TGSPLIT-NEXT: s_endpgm
8319 ;
8320 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8321 ; GFX11-WGP: ; %bb.0: ; %entry
8322 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8323 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8324 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8325 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8326 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8327 ; GFX11-WGP-NEXT: s_endpgm
8328 ;
8329 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
8330 ; GFX11-CU: ; %bb.0: ; %entry
8331 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8332 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8333 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8334 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8335 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8336 ; GFX11-CU-NEXT: s_endpgm
8337 ptr %out, i32 %in, i32 %old) {
8338 entry:
8339 %gep = getelementptr i32, ptr %out, i32 4
8340 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
8341 ret void
8342 }
8344 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
8345 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8346 ; GFX7: ; %bb.0: ; %entry
8347 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8348 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8349 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8350 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8351 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8352 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8353 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8354 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8355 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8356 ; GFX7-NEXT: s_endpgm
8357 ;
8358 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8359 ; GFX10-WGP: ; %bb.0: ; %entry
8360 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8361 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8362 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8363 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8364 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8365 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8366 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8367 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8368 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8369 ; GFX10-WGP-NEXT: s_endpgm
8370 ;
8371 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8372 ; GFX10-CU: ; %bb.0: ; %entry
8373 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8374 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8375 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8376 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8377 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8378 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8379 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8380 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8381 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8382 ; GFX10-CU-NEXT: s_endpgm
8383 ;
8384 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8385 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8386 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8387 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8388 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8389 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8390 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8392 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8393 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8394 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8395 ; SKIP-CACHE-INV-NEXT: s_endpgm
8396 ;
8397 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8398 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8399 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8400 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8401 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8402 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8403 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8404 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8405 ;
8406 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8407 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8408 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8409 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8410 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8411 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8412 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8413 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8414 ;
8415 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8416 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8417 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8418 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8419 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8420 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8421 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8422 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8423 ;
8424 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8425 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8426 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8427 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8428 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8429 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8430 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8431 ; GFX940-TGSPLIT-NEXT: s_endpgm
8432 ;
8433 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8434 ; GFX11-WGP: ; %bb.0: ; %entry
8435 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8436 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8437 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8438 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8439 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8440 ; GFX11-WGP-NEXT: s_endpgm
8441 ;
8442 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
8443 ; GFX11-CU: ; %bb.0: ; %entry
8444 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8445 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8446 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8447 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8448 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8449 ; GFX11-CU-NEXT: s_endpgm
8450 ptr %out, i32 %in, i32 %old) {
8451 entry:
8452 %gep = getelementptr i32, ptr %out, i32 4
8453 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic acquire
8454 ret void
8455 }
8457 define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
8458 ; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8459 ; GFX7: ; %bb.0: ; %entry
8460 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8461 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8462 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8463 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8464 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8465 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8466 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8467 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8468 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8469 ; GFX7-NEXT: s_endpgm
8470 ;
8471 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8472 ; GFX10-WGP: ; %bb.0: ; %entry
8473 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8474 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8475 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8476 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8477 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8478 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8479 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8480 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8481 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8482 ; GFX10-WGP-NEXT: s_endpgm
8483 ;
8484 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8485 ; GFX10-CU: ; %bb.0: ; %entry
8486 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8487 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8488 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8489 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8490 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8491 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8492 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8493 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8494 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8495 ; GFX10-CU-NEXT: s_endpgm
8496 ;
8497 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8498 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8499 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8500 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8501 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8502 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8503 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8504 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8505 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8506 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8507 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8508 ; SKIP-CACHE-INV-NEXT: s_endpgm
8509 ;
8510 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8511 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8512 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8513 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8514 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8515 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8516 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8517 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8518 ;
8519 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8520 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8521 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8522 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8523 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8524 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8525 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8526 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8527 ;
8528 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8529 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8530 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8531 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8532 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8533 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8534 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8535 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8536 ;
8537 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8538 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8539 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8540 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8541 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8542 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8543 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8544 ; GFX940-TGSPLIT-NEXT: s_endpgm
8545 ;
8546 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8547 ; GFX11-WGP: ; %bb.0: ; %entry
8548 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8549 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8550 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8551 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8552 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8553 ; GFX11-WGP-NEXT: s_endpgm
8554 ;
8555 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
8556 ; GFX11-CU: ; %bb.0: ; %entry
8557 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8558 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8559 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8560 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8561 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8562 ; GFX11-CU-NEXT: s_endpgm
8563 ptr %out, i32 %in, i32 %old) {
8564 entry:
8565 %gep = getelementptr i32, ptr %out, i32 4
8566 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
8567 ret void
8568 }
8570 define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
8571 ; GFX7-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8572 ; GFX7: ; %bb.0: ; %entry
8573 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8574 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8575 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8576 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8577 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8578 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8579 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8580 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8581 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8582 ; GFX7-NEXT: s_endpgm
8583 ;
8584 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8585 ; GFX10-WGP: ; %bb.0: ; %entry
8586 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8587 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8588 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8589 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8590 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8591 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8592 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8593 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8594 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8595 ; GFX10-WGP-NEXT: s_endpgm
8596 ;
8597 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8598 ; GFX10-CU: ; %bb.0: ; %entry
8599 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8600 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8601 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8602 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8603 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8604 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8605 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8606 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8607 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8608 ; GFX10-CU-NEXT: s_endpgm
8609 ;
8610 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8611 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8612 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8613 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8614 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8615 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8616 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8617 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8618 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8619 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8620 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8621 ; SKIP-CACHE-INV-NEXT: s_endpgm
8622 ;
8623 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8624 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8625 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8626 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8627 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8628 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8629 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8630 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8631 ;
8632 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8633 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8634 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8635 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8636 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8637 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8638 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8639 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8640 ;
8641 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8642 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8643 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8644 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8645 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8646 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8647 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8648 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8649 ;
8650 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8651 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8652 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8653 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8654 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8655 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8656 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8657 ; GFX940-TGSPLIT-NEXT: s_endpgm
8658 ;
8659 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8660 ; GFX11-WGP: ; %bb.0: ; %entry
8661 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8662 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8663 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8664 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8665 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8666 ; GFX11-WGP-NEXT: s_endpgm
8667 ;
8668 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
8669 ; GFX11-CU: ; %bb.0: ; %entry
8670 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8671 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8672 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8673 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8674 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8675 ; GFX11-CU-NEXT: s_endpgm
8676 ptr %out, i32 %in, i32 %old) {
8677 entry:
8678 %gep = getelementptr i32, ptr %out, i32 4
8679 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
8680 ret void
8681 }
8683 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
8684 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8685 ; GFX7: ; %bb.0: ; %entry
8686 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8687 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8688 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8689 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8690 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8691 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8692 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8693 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8694 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8695 ; GFX7-NEXT: s_endpgm
8696 ;
8697 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8698 ; GFX10-WGP: ; %bb.0: ; %entry
8699 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8700 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8701 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8702 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8703 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8704 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8705 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8706 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8707 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8708 ; GFX10-WGP-NEXT: s_endpgm
8709 ;
8710 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8711 ; GFX10-CU: ; %bb.0: ; %entry
8712 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8713 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8714 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8715 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8716 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8717 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8718 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8719 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8720 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8721 ; GFX10-CU-NEXT: s_endpgm
8722 ;
8723 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8724 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8725 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8726 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8727 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8728 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8729 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8730 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8731 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8733 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8734 ; SKIP-CACHE-INV-NEXT: s_endpgm
8735 ;
8736 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8737 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8738 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8739 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8740 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8741 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8742 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8743 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8744 ;
8745 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8746 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8747 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8748 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8749 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8750 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8751 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8752 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8753 ;
8754 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8755 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8756 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8757 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8758 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8759 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8760 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8761 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8762 ;
8763 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8764 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8765 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8766 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8767 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8768 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8769 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8770 ; GFX940-TGSPLIT-NEXT: s_endpgm
8771 ;
8772 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8773 ; GFX11-WGP: ; %bb.0: ; %entry
8774 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8775 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8776 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8777 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8778 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8779 ; GFX11-WGP-NEXT: s_endpgm
8780 ;
8781 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
8782 ; GFX11-CU: ; %bb.0: ; %entry
8783 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8784 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8785 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8786 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8787 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8788 ; GFX11-CU-NEXT: s_endpgm
8789 ptr %out, i32 %in, i32 %old) {
8790 entry:
8791 %gep = getelementptr i32, ptr %out, i32 4
8792 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
8793 ret void
8794 }
8796 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
8797 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8798 ; GFX7: ; %bb.0: ; %entry
8799 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8800 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8801 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8802 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8803 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8804 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8805 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8806 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8807 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8808 ; GFX7-NEXT: s_endpgm
8809 ;
8810 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8811 ; GFX10-WGP: ; %bb.0: ; %entry
8812 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8813 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8814 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8815 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8816 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8817 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8818 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8819 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8820 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8821 ; GFX10-WGP-NEXT: s_endpgm
8822 ;
8823 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8824 ; GFX10-CU: ; %bb.0: ; %entry
8825 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8826 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8827 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8828 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8829 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8830 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8831 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8832 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8833 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8834 ; GFX10-CU-NEXT: s_endpgm
8835 ;
8836 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8837 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8838 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8839 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8840 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8841 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8842 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8843 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8844 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8845 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8846 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8847 ; SKIP-CACHE-INV-NEXT: s_endpgm
8848 ;
8849 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8850 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8851 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8852 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8853 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8854 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8855 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8856 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8857 ;
8858 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8859 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8860 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8861 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8862 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8863 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8864 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8865 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8866 ;
8867 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8868 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8869 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8870 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8871 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8872 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8873 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8874 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8875 ;
8876 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8877 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8878 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8879 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8880 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8881 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8882 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8883 ; GFX940-TGSPLIT-NEXT: s_endpgm
8884 ;
8885 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8886 ; GFX11-WGP: ; %bb.0: ; %entry
8887 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8888 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8889 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8890 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8891 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8892 ; GFX11-WGP-NEXT: s_endpgm
8893 ;
8894 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
8895 ; GFX11-CU: ; %bb.0: ; %entry
8896 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
8897 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8898 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
8899 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
8900 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
8901 ; GFX11-CU-NEXT: s_endpgm
8902 ptr %out, i32 %in, i32 %old) {
8903 entry:
8904 %gep = getelementptr i32, ptr %out, i32 4
8905 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
8906 ret void
8907 }
8909 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
8910 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8911 ; GFX7: ; %bb.0: ; %entry
8912 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8913 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8914 ; GFX7-NEXT: s_add_u32 s0, s0, 16
8915 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
8916 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
8917 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
8918 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
8919 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
8920 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8921 ; GFX7-NEXT: s_endpgm
8922 ;
8923 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8924 ; GFX10-WGP: ; %bb.0: ; %entry
8925 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8926 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8927 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
8928 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
8929 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
8930 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
8931 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
8932 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
8933 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8934 ; GFX10-WGP-NEXT: s_endpgm
8935 ;
8936 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8937 ; GFX10-CU: ; %bb.0: ; %entry
8938 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8939 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8940 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
8941 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
8942 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
8943 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
8944 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
8945 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
8946 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8947 ; GFX10-CU-NEXT: s_endpgm
8948 ;
8949 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8950 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8951 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8952 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8953 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
8954 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
8955 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
8956 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
8957 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
8958 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
8959 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
8960 ; SKIP-CACHE-INV-NEXT: s_endpgm
8961 ;
8962 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8963 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8964 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8965 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8966 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8967 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8968 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8969 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8970 ;
8971 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8972 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8973 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
8974 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8975 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
8976 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
8977 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8978 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8979 ;
8980 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8981 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8982 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8983 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8984 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8985 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8986 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8987 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8988 ;
8989 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8990 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8991 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
8992 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8993 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
8994 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
8995 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
8996 ; GFX940-TGSPLIT-NEXT: s_endpgm
8997 ;
8998 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
8999 ; GFX11-WGP: ; %bb.0: ; %entry
9000 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9001 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9002 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9003 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9004 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9005 ; GFX11-WGP-NEXT: s_endpgm
9006 ;
9007 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
9008 ; GFX11-CU: ; %bb.0: ; %entry
9009 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9010 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9011 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9012 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9013 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9014 ; GFX11-CU-NEXT: s_endpgm
9015 ptr %out, i32 %in, i32 %old) {
9016 entry:
9017 %gep = getelementptr i32, ptr %out, i32 4
9018 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic seq_cst
9019 ret void
9020 }
9022 define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
9023 ; GFX7-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9024 ; GFX7: ; %bb.0: ; %entry
9025 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9026 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9027 ; GFX7-NEXT: s_add_u32 s0, s0, 16
9028 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
9029 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9030 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9031 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9032 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9033 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9034 ; GFX7-NEXT: s_endpgm
9035 ;
9036 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9037 ; GFX10-WGP: ; %bb.0: ; %entry
9038 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9039 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9040 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
9041 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
9042 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9043 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9044 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9045 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9046 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9047 ; GFX10-WGP-NEXT: s_endpgm
9048 ;
9049 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9050 ; GFX10-CU: ; %bb.0: ; %entry
9051 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9052 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9053 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
9054 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
9055 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9056 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9057 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9058 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9059 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9060 ; GFX10-CU-NEXT: s_endpgm
9061 ;
9062 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9063 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9064 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9065 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9066 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
9067 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
9068 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9069 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9070 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9071 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9072 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9073 ; SKIP-CACHE-INV-NEXT: s_endpgm
9074 ;
9075 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9076 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9077 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9078 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9079 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9080 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9081 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9082 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9083 ;
9084 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9085 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9086 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9087 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9088 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9089 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9090 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9091 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9092 ;
9093 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9094 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9095 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9096 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9097 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9098 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9099 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9100 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9101 ;
9102 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9103 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9104 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9105 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9106 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9107 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9108 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9109 ; GFX940-TGSPLIT-NEXT: s_endpgm
9110 ;
9111 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9112 ; GFX11-WGP: ; %bb.0: ; %entry
9113 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9114 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9115 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9116 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9117 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9118 ; GFX11-WGP-NEXT: s_endpgm
9119 ;
9120 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
9121 ; GFX11-CU: ; %bb.0: ; %entry
9122 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9123 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9124 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9125 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9126 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9127 ; GFX11-CU-NEXT: s_endpgm
9128 ptr %out, i32 %in, i32 %old) {
9129 entry:
9130 %gep = getelementptr i32, ptr %out, i32 4
9131 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire seq_cst
9132 ret void
9133 }
9135 define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
9136 ; GFX7-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9137 ; GFX7: ; %bb.0: ; %entry
9138 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9139 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9140 ; GFX7-NEXT: s_add_u32 s0, s0, 16
9141 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
9142 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9143 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9144 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9145 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9146 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9147 ; GFX7-NEXT: s_endpgm
9148 ;
9149 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9150 ; GFX10-WGP: ; %bb.0: ; %entry
9151 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9152 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9153 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
9154 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
9155 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9156 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9157 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9158 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9159 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9160 ; GFX10-WGP-NEXT: s_endpgm
9161 ;
9162 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9163 ; GFX10-CU: ; %bb.0: ; %entry
9164 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9165 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9166 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
9167 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
9168 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9169 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9170 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9171 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9172 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9173 ; GFX10-CU-NEXT: s_endpgm
9174 ;
9175 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9176 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9177 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9178 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9179 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
9180 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
9181 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9182 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9183 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9184 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9185 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9186 ; SKIP-CACHE-INV-NEXT: s_endpgm
9187 ;
9188 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9189 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9190 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9191 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9192 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9193 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9194 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9195 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9196 ;
9197 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9198 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9199 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9200 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9201 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9202 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9203 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9204 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9205 ;
9206 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9207 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9208 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9209 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9210 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9211 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9212 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9213 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9214 ;
9215 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9216 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9217 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9218 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9219 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9220 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9221 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9222 ; GFX940-TGSPLIT-NEXT: s_endpgm
9223 ;
9224 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9225 ; GFX11-WGP: ; %bb.0: ; %entry
9226 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9227 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9228 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9229 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9230 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9231 ; GFX11-WGP-NEXT: s_endpgm
9232 ;
9233 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
9234 ; GFX11-CU: ; %bb.0: ; %entry
9235 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9236 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9237 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9238 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9239 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9240 ; GFX11-CU-NEXT: s_endpgm
9241 ptr %out, i32 %in, i32 %old) {
9242 entry:
9243 %gep = getelementptr i32, ptr %out, i32 4
9244 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release seq_cst
9245 ret void
9246 }
9248 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
9249 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9250 ; GFX7: ; %bb.0: ; %entry
9251 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9252 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9253 ; GFX7-NEXT: s_add_u32 s0, s0, 16
9254 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
9255 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9256 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9257 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9258 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9259 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9260 ; GFX7-NEXT: s_endpgm
9261 ;
9262 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9263 ; GFX10-WGP: ; %bb.0: ; %entry
9264 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9265 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9266 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
9267 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
9268 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9270 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9271 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9272 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9273 ; GFX10-WGP-NEXT: s_endpgm
9274 ;
9275 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9276 ; GFX10-CU: ; %bb.0: ; %entry
9277 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9278 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9279 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
9280 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
9281 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9282 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9283 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9284 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9285 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9286 ; GFX10-CU-NEXT: s_endpgm
9287 ;
9288 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9289 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9290 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9291 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9292 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
9293 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
9294 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9295 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9296 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9297 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9298 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9299 ; SKIP-CACHE-INV-NEXT: s_endpgm
9300 ;
9301 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9302 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9303 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9304 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9305 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9306 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9307 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9308 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9309 ;
9310 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9311 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9312 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9313 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9314 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9315 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9316 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9317 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9318 ;
9319 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9320 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9321 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9322 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9323 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9324 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9325 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9326 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9327 ;
9328 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9329 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9330 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9331 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9332 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9333 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9334 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9335 ; GFX940-TGSPLIT-NEXT: s_endpgm
9336 ;
9337 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9338 ; GFX11-WGP: ; %bb.0: ; %entry
9339 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9340 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9341 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9342 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9343 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9344 ; GFX11-WGP-NEXT: s_endpgm
9345 ;
9346 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
9347 ; GFX11-CU: ; %bb.0: ; %entry
9348 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9349 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9350 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9351 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9352 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9353 ; GFX11-CU-NEXT: s_endpgm
9354 ptr %out, i32 %in, i32 %old) {
9355 entry:
9356 %gep = getelementptr i32, ptr %out, i32 4
9357 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel seq_cst
9358 ret void
9359 }
9361 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
9362 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9363 ; GFX7: ; %bb.0: ; %entry
9364 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9365 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9366 ; GFX7-NEXT: s_add_u32 s0, s0, 16
9367 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
9368 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9369 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9370 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9371 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9372 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9373 ; GFX7-NEXT: s_endpgm
9374 ;
9375 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9376 ; GFX10-WGP: ; %bb.0: ; %entry
9377 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9378 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9379 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, 16
9380 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
9381 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9382 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9383 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9384 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9385 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9386 ; GFX10-WGP-NEXT: s_endpgm
9387 ;
9388 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9389 ; GFX10-CU: ; %bb.0: ; %entry
9390 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9391 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9392 ; GFX10-CU-NEXT: s_add_u32 s0, s0, 16
9393 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
9394 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9395 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9396 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9397 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9398 ; GFX10-CU-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9399 ; GFX10-CU-NEXT: s_endpgm
9400 ;
9401 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9402 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9403 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9404 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9405 ; SKIP-CACHE-INV-NEXT: s_add_u32 s0, s0, 16
9406 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s1, s1, 0
9407 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9408 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9409 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9410 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9411 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
9412 ; SKIP-CACHE-INV-NEXT: s_endpgm
9413 ;
9414 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9415 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9416 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9417 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9418 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9419 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9420 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9421 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9422 ;
9423 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9424 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9425 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9426 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9427 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9428 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9429 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9430 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9431 ;
9432 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9433 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9434 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9435 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9436 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9437 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9438 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9439 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9440 ;
9441 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9442 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9443 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9444 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9445 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9446 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9447 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
9448 ; GFX940-TGSPLIT-NEXT: s_endpgm
9449 ;
9450 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9451 ; GFX11-WGP: ; %bb.0: ; %entry
9452 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9453 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9454 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9455 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9456 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9457 ; GFX11-WGP-NEXT: s_endpgm
9458 ;
9459 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
9460 ; GFX11-CU: ; %bb.0: ; %entry
9461 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9462 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9463 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9464 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9465 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
9466 ; GFX11-CU-NEXT: s_endpgm
9467 ptr %out, i32 %in, i32 %old) {
9468 entry:
9469 %gep = getelementptr i32, ptr %out, i32 4
9470 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
9471 ret void
9472 }
9474 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg(
9475 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9476 ; GFX7: ; %bb.0: ; %entry
9477 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9478 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9479 ; GFX7-NEXT: s_add_u32 s4, s0, 16
9480 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
9481 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9482 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9483 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9484 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9485 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9486 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9487 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9488 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9489 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9490 ; GFX7-NEXT: s_endpgm
9491 ;
9492 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9493 ; GFX10-WGP: ; %bb.0: ; %entry
9494 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9495 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9496 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
9497 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
9498 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9499 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9500 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9501 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9502 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9503 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9504 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9505 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9506 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9507 ; GFX10-WGP-NEXT: s_endpgm
9508 ;
9509 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9510 ; GFX10-CU: ; %bb.0: ; %entry
9511 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9512 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9513 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
9514 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
9515 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9516 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9517 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9518 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9519 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9520 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9521 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9522 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9523 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9524 ; GFX10-CU-NEXT: s_endpgm
9525 ;
9526 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9527 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9528 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9529 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9530 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
9531 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
9532 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9533 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9534 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
9535 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9536 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9537 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9538 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9539 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9540 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9541 ; SKIP-CACHE-INV-NEXT: s_endpgm
9542 ;
9543 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9544 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9545 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9546 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9547 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9548 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9549 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9550 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9551 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9552 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9553 ;
9554 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9555 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9556 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9557 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9558 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9559 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9560 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9561 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9562 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9563 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9564 ;
9565 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9566 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9567 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9568 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9569 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9570 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9571 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9572 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9573 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9574 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9575 ;
9576 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9577 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9578 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9579 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9580 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9581 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9582 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9583 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9584 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9585 ; GFX940-TGSPLIT-NEXT: s_endpgm
9586 ;
9587 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9588 ; GFX11-WGP: ; %bb.0: ; %entry
9589 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9590 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9591 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9592 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9593 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9594 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9595 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9596 ; GFX11-WGP-NEXT: s_endpgm
9597 ;
9598 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
9599 ; GFX11-CU: ; %bb.0: ; %entry
9600 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9601 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9602 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9603 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9604 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9605 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9606 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9607 ; GFX11-CU-NEXT: s_endpgm
9608 ptr %out, i32 %in, i32 %old) {
9609 entry:
9610 %gep = getelementptr i32, ptr %out, i32 4
9611 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
9612 %val0 = extractvalue { i32, i1 } %val, 0
9613 store i32 %val0, ptr %out, align 4
9614 ret void
9615 }
9617 define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
9618 ; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9619 ; GFX7: ; %bb.0: ; %entry
9620 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9621 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9622 ; GFX7-NEXT: s_add_u32 s4, s0, 16
9623 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
9624 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9625 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9626 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9627 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9628 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9629 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9630 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9631 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9632 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9633 ; GFX7-NEXT: s_endpgm
9634 ;
9635 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9636 ; GFX10-WGP: ; %bb.0: ; %entry
9637 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9638 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9639 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
9640 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
9641 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9642 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9643 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9644 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9645 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9646 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9647 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9648 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9649 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9650 ; GFX10-WGP-NEXT: s_endpgm
9651 ;
9652 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9653 ; GFX10-CU: ; %bb.0: ; %entry
9654 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9655 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9656 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
9657 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
9658 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9659 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9660 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9661 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9662 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9663 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9664 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9665 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9666 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9667 ; GFX10-CU-NEXT: s_endpgm
9668 ;
9669 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9670 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9671 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9672 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9673 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
9674 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
9675 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9676 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9677 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
9678 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9679 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9680 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9681 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9682 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9683 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9684 ; SKIP-CACHE-INV-NEXT: s_endpgm
9685 ;
9686 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9687 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9688 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9689 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9690 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9691 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9692 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9693 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9694 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9695 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9696 ;
9697 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9698 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9699 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9700 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9701 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9702 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9703 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9704 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9705 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9706 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9707 ;
9708 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9709 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9710 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9711 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9712 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9713 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9714 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9715 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9716 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9717 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9718 ;
9719 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9720 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9721 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9722 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9723 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9724 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9725 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9726 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9727 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9728 ; GFX940-TGSPLIT-NEXT: s_endpgm
9729 ;
9730 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9731 ; GFX11-WGP: ; %bb.0: ; %entry
9732 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9733 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9734 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9735 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9736 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9737 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9738 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9739 ; GFX11-WGP-NEXT: s_endpgm
9740 ;
9741 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
9742 ; GFX11-CU: ; %bb.0: ; %entry
9743 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9744 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9745 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9746 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9747 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9748 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9749 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9750 ; GFX11-CU-NEXT: s_endpgm
9751 ptr %out, i32 %in, i32 %old) {
9752 entry:
9753 %gep = getelementptr i32, ptr %out, i32 4
9754 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
9755 %val0 = extractvalue { i32, i1 } %val, 0
9756 store i32 %val0, ptr %out, align 4
9757 ret void
9758 }
9760 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
9761 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9762 ; GFX7: ; %bb.0: ; %entry
9763 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9764 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9765 ; GFX7-NEXT: s_add_u32 s4, s0, 16
9766 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
9767 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9768 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9769 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9770 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9771 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9772 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9773 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9774 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9775 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9776 ; GFX7-NEXT: s_endpgm
9777 ;
9778 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9779 ; GFX10-WGP: ; %bb.0: ; %entry
9780 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9781 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9782 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
9783 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
9784 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9785 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9786 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9788 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9789 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9791 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9792 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9793 ; GFX10-WGP-NEXT: s_endpgm
9794 ;
9795 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9796 ; GFX10-CU: ; %bb.0: ; %entry
9797 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9798 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9799 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
9800 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
9801 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9802 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9803 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9804 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9805 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9806 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9807 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9808 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9809 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9810 ; GFX10-CU-NEXT: s_endpgm
9811 ;
9812 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9813 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9814 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9815 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9816 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
9817 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
9818 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9819 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9820 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
9821 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9822 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9823 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9824 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9825 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9826 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9827 ; SKIP-CACHE-INV-NEXT: s_endpgm
9828 ;
9829 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9830 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9831 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9832 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9833 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9834 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9835 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9836 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9837 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9838 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9839 ;
9840 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9841 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9842 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9843 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9844 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9845 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9846 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9847 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9848 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9849 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9850 ;
9851 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9852 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9853 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9854 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9855 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9856 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9857 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9858 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9859 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9860 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9861 ;
9862 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9863 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9864 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9865 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9866 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9867 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
9868 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
9869 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9870 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
9871 ; GFX940-TGSPLIT-NEXT: s_endpgm
9872 ;
9873 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9874 ; GFX11-WGP: ; %bb.0: ; %entry
9875 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9876 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9877 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9878 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9879 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9880 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9881 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
9882 ; GFX11-WGP-NEXT: s_endpgm
9883 ;
9884 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
9885 ; GFX11-CU: ; %bb.0: ; %entry
9886 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
9887 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9888 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
9889 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
9890 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
9891 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9892 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
9893 ; GFX11-CU-NEXT: s_endpgm
9894 ptr %out, i32 %in, i32 %old) {
9895 entry:
9896 %gep = getelementptr i32, ptr %out, i32 4
9897 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
9898 %val0 = extractvalue { i32, i1 } %val, 0
9899 store i32 %val0, ptr %out, align 4
9900 ret void
9901 }
9903 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
9904 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9905 ; GFX7: ; %bb.0: ; %entry
9906 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9907 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9908 ; GFX7-NEXT: s_add_u32 s4, s0, 16
9909 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
9910 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9911 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
9912 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9913 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
9914 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9915 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
9916 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
9917 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9918 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9919 ; GFX7-NEXT: s_endpgm
9920 ;
9921 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9922 ; GFX10-WGP: ; %bb.0: ; %entry
9923 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9924 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9925 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
9926 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
9927 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
9928 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
9929 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
9930 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
9931 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9932 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
9933 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
9934 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9935 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
9936 ; GFX10-WGP-NEXT: s_endpgm
9937 ;
9938 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9939 ; GFX10-CU: ; %bb.0: ; %entry
9940 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9941 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9942 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
9943 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
9944 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
9945 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
9946 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
9947 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
9948 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9949 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
9950 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
9951 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9952 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
9953 ; GFX10-CU-NEXT: s_endpgm
9954 ;
9955 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9956 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9957 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9958 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9959 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
9960 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
9961 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
9962 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
9963 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
9964 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
9965 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9966 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
9967 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
9968 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9969 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
9970 ; SKIP-CACHE-INV-NEXT: s_endpgm
9971 ;
9972 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9973 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9974 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9975 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9976 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9977 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9978 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9979 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9980 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
9981 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9982 ;
9983 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9984 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9985 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
9986 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9987 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
9988 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
9989 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
9990 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9991 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
9992 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9993 ;
9994 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
9995 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9996 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
9997 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9998 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
9999 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10000 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10001 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10002 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10003 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10004 ;
10005 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
10006 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10007 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10008 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10009 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10010 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10011 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10012 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10013 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10014 ; GFX940-TGSPLIT-NEXT: s_endpgm
10015 ;
10016 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
10017 ; GFX11-WGP: ; %bb.0: ; %entry
10018 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10019 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10020 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10021 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10022 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10023 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10024 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10025 ; GFX11-WGP-NEXT: s_endpgm
10026 ;
10027 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
10028 ; GFX11-CU: ; %bb.0: ; %entry
10029 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10030 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10031 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10032 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10033 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10034 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10035 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10036 ; GFX11-CU-NEXT: s_endpgm
10037 ptr %out, i32 %in, i32 %old) {
10038 entry:
10039 %gep = getelementptr i32, ptr %out, i32 4
10040 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
10041 %val0 = extractvalue { i32, i1 } %val, 0
10042 store i32 %val0, ptr %out, align 4
10043 ret void
10044 }
10046 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
10047 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10048 ; GFX7: ; %bb.0: ; %entry
10049 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10050 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10051 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10052 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10053 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10054 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10055 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10056 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10057 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10058 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10059 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10060 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10061 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10062 ; GFX7-NEXT: s_endpgm
10063 ;
10064 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10065 ; GFX10-WGP: ; %bb.0: ; %entry
10066 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10067 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10068 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10069 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10070 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10071 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10072 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10073 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10074 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10075 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10076 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10077 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10078 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10079 ; GFX10-WGP-NEXT: s_endpgm
10080 ;
10081 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10082 ; GFX10-CU: ; %bb.0: ; %entry
10083 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10084 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10085 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10086 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10087 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10088 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10089 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10090 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10091 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10092 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10093 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10094 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10095 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10096 ; GFX10-CU-NEXT: s_endpgm
10097 ;
10098 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10099 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10100 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10101 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10102 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10103 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10104 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10105 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10106 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10107 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10108 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10109 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10110 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10111 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10112 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10113 ; SKIP-CACHE-INV-NEXT: s_endpgm
10114 ;
10115 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10116 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10117 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10118 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10119 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10120 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10121 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10122 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10123 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10124 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10125 ;
10126 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10127 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10128 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10129 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10130 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10131 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10132 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10133 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10134 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10135 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10136 ;
10137 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10138 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10139 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10140 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10141 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10142 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10143 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10144 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10145 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10146 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10147 ;
10148 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10149 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10150 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10151 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10152 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10153 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10154 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10155 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10156 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10157 ; GFX940-TGSPLIT-NEXT: s_endpgm
10158 ;
10159 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10160 ; GFX11-WGP: ; %bb.0: ; %entry
10161 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10162 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10163 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10164 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10165 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10166 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10167 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10168 ; GFX11-WGP-NEXT: s_endpgm
10169 ;
10170 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
10171 ; GFX11-CU: ; %bb.0: ; %entry
10172 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10173 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10174 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10175 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10176 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10177 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10178 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10179 ; GFX11-CU-NEXT: s_endpgm
10180 ptr %out, i32 %in, i32 %old) {
10181 entry:
10182 %gep = getelementptr i32, ptr %out, i32 4
10183 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic acquire
10184 %val0 = extractvalue { i32, i1 } %val, 0
10185 store i32 %val0, ptr %out, align 4
10186 ret void
10187 }
10189 define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
10190 ; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10191 ; GFX7: ; %bb.0: ; %entry
10192 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10193 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10194 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10195 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10196 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10197 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10198 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10199 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10200 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10201 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10202 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10203 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10204 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10205 ; GFX7-NEXT: s_endpgm
10206 ;
10207 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10208 ; GFX10-WGP: ; %bb.0: ; %entry
10209 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10210 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10211 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10212 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10213 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10214 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10215 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10216 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10217 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10218 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10219 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10220 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10221 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10222 ; GFX10-WGP-NEXT: s_endpgm
10223 ;
10224 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10225 ; GFX10-CU: ; %bb.0: ; %entry
10226 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10227 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10228 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10229 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10230 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10231 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10232 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10233 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10234 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10235 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10236 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10237 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10238 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10239 ; GFX10-CU-NEXT: s_endpgm
10240 ;
10241 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10242 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10243 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10244 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10245 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10246 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10247 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10249 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10250 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10251 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10252 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10253 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10254 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10255 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10256 ; SKIP-CACHE-INV-NEXT: s_endpgm
10257 ;
10258 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10259 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10260 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10261 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10262 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10263 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10264 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10265 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10266 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10267 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10268 ;
10269 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10270 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10271 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10272 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10273 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10274 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10275 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10276 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10277 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10278 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10279 ;
10280 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10281 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10282 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10283 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10284 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10285 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10286 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10287 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10288 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10289 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10290 ;
10291 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10292 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10293 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10294 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10295 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10296 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10297 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10298 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10299 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10300 ; GFX940-TGSPLIT-NEXT: s_endpgm
10301 ;
10302 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10303 ; GFX11-WGP: ; %bb.0: ; %entry
10304 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10305 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10306 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10307 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10308 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10309 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10310 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10311 ; GFX11-WGP-NEXT: s_endpgm
10312 ;
10313 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
10314 ; GFX11-CU: ; %bb.0: ; %entry
10315 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10316 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10317 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10318 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10319 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10320 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10321 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10322 ; GFX11-CU-NEXT: s_endpgm
10323 ptr %out, i32 %in, i32 %old) {
10324 entry:
10325 %gep = getelementptr i32, ptr %out, i32 4
10326 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
10327 %val0 = extractvalue { i32, i1 } %val, 0
10328 store i32 %val0, ptr %out, align 4
10329 ret void
10330 }
10332 define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
10333 ; GFX7-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10334 ; GFX7: ; %bb.0: ; %entry
10335 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10336 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10337 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10338 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10339 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10340 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10341 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10342 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10343 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10344 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10345 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10346 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10347 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10348 ; GFX7-NEXT: s_endpgm
10349 ;
10350 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10351 ; GFX10-WGP: ; %bb.0: ; %entry
10352 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10353 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10354 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10355 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10356 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10357 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10358 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10359 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10360 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10361 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10362 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10363 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10364 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10365 ; GFX10-WGP-NEXT: s_endpgm
10366 ;
10367 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10368 ; GFX10-CU: ; %bb.0: ; %entry
10369 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10370 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10371 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10372 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10373 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10374 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10375 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10376 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10377 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10378 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10379 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10380 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10381 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10382 ; GFX10-CU-NEXT: s_endpgm
10383 ;
10384 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10385 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10386 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10387 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10388 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10389 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10390 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10392 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10393 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10394 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10395 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10396 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10397 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10398 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10399 ; SKIP-CACHE-INV-NEXT: s_endpgm
10400 ;
10401 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10402 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10403 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10404 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10405 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10406 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10407 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10408 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10409 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10410 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10411 ;
10412 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10413 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10414 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10415 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10416 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10417 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10418 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10419 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10420 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10421 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10422 ;
10423 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10424 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10425 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10426 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10427 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10428 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10429 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10430 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10431 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10432 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10433 ;
10434 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10435 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10436 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10437 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10438 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10439 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10440 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10441 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10442 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10443 ; GFX940-TGSPLIT-NEXT: s_endpgm
10444 ;
10445 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10446 ; GFX11-WGP: ; %bb.0: ; %entry
10447 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10448 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10449 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10450 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10451 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10452 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10453 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10454 ; GFX11-WGP-NEXT: s_endpgm
10455 ;
10456 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
10457 ; GFX11-CU: ; %bb.0: ; %entry
10458 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10459 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10460 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10461 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10462 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10463 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10464 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10465 ; GFX11-CU-NEXT: s_endpgm
10466 ptr %out, i32 %in, i32 %old) {
10467 entry:
10468 %gep = getelementptr i32, ptr %out, i32 4
10469 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
10470 %val0 = extractvalue { i32, i1 } %val, 0
10471 store i32 %val0, ptr %out, align 4
10472 ret void
10473 }
10475 define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
10476 ; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10477 ; GFX7: ; %bb.0: ; %entry
10478 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10479 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10480 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10481 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10482 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10483 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10484 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10485 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10486 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10487 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10488 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10489 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10490 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10491 ; GFX7-NEXT: s_endpgm
10492 ;
10493 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10494 ; GFX10-WGP: ; %bb.0: ; %entry
10495 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10496 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10497 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10498 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10499 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10500 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10501 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10502 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10503 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10504 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10505 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10506 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10507 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10508 ; GFX10-WGP-NEXT: s_endpgm
10509 ;
10510 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10511 ; GFX10-CU: ; %bb.0: ; %entry
10512 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10513 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10514 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10515 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10516 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10517 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10518 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10519 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10520 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10521 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10522 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10523 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10524 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10525 ; GFX10-CU-NEXT: s_endpgm
10526 ;
10527 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10528 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10529 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10530 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10531 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10532 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10533 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10534 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10535 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10536 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10537 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10538 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10539 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10540 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10541 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10542 ; SKIP-CACHE-INV-NEXT: s_endpgm
10543 ;
10544 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10545 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10546 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10547 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10548 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10549 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10550 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10551 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10552 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10553 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10554 ;
10555 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10556 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10557 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10558 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10559 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10560 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10561 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10562 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10563 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10564 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10565 ;
10566 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10567 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10568 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10569 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10570 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10571 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10572 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10573 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10574 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10575 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10576 ;
10577 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10578 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10579 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10580 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10581 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10582 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10583 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10584 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10585 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10586 ; GFX940-TGSPLIT-NEXT: s_endpgm
10587 ;
10588 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10589 ; GFX11-WGP: ; %bb.0: ; %entry
10590 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10591 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10592 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10593 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10594 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10595 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10596 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10597 ; GFX11-WGP-NEXT: s_endpgm
10598 ;
10599 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
10600 ; GFX11-CU: ; %bb.0: ; %entry
10601 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10602 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10603 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10604 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10605 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10606 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10607 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10608 ; GFX11-CU-NEXT: s_endpgm
10609 ptr %out, i32 %in, i32 %old) {
10610 entry:
10611 %gep = getelementptr i32, ptr %out, i32 4
10612 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
10613 %val0 = extractvalue { i32, i1 } %val, 0
10614 store i32 %val0, ptr %out, align 4
10615 ret void
10616 }
10618 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
10619 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10620 ; GFX7: ; %bb.0: ; %entry
10621 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10622 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10623 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10624 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10625 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10626 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10627 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10628 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10629 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10630 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10631 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10632 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10633 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10634 ; GFX7-NEXT: s_endpgm
10635 ;
10636 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10637 ; GFX10-WGP: ; %bb.0: ; %entry
10638 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10639 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10640 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10641 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10642 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10643 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10644 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10645 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10646 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10647 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10648 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10649 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10650 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10651 ; GFX10-WGP-NEXT: s_endpgm
10652 ;
10653 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10654 ; GFX10-CU: ; %bb.0: ; %entry
10655 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10656 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10657 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10658 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10659 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10660 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10661 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10662 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10663 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10664 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10665 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10666 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10667 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10668 ; GFX10-CU-NEXT: s_endpgm
10669 ;
10670 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10671 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10672 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10673 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10674 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10675 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10676 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10677 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10678 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10679 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10680 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10681 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10682 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10683 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10684 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10685 ; SKIP-CACHE-INV-NEXT: s_endpgm
10686 ;
10687 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10688 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10689 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10690 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10691 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10692 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10693 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10694 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10695 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10696 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10697 ;
10698 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10699 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10700 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10701 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10702 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10703 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10704 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10705 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10706 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10707 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10708 ;
10709 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10710 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10711 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10712 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10713 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10714 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10715 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10716 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10717 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10718 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10719 ;
10720 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10721 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10722 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10723 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10724 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10725 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10726 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10727 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10728 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10729 ; GFX940-TGSPLIT-NEXT: s_endpgm
10730 ;
10731 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10732 ; GFX11-WGP: ; %bb.0: ; %entry
10733 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10734 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10735 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10736 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10737 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10738 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10739 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10740 ; GFX11-WGP-NEXT: s_endpgm
10741 ;
10742 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
10743 ; GFX11-CU: ; %bb.0: ; %entry
10744 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10745 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10746 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10747 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10748 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10749 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10750 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10751 ; GFX11-CU-NEXT: s_endpgm
10752 ptr %out, i32 %in, i32 %old) {
10753 entry:
10754 %gep = getelementptr i32, ptr %out, i32 4
10755 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
10756 %val0 = extractvalue { i32, i1 } %val, 0
10757 store i32 %val0, ptr %out, align 4
10758 ret void
10759 }
10761 define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
10762 ; GFX7-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10763 ; GFX7: ; %bb.0: ; %entry
10764 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10765 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10766 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10767 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10768 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10769 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10770 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10771 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10772 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10773 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10774 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10775 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10776 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10777 ; GFX7-NEXT: s_endpgm
10778 ;
10779 ; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10780 ; GFX10-WGP: ; %bb.0: ; %entry
10781 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10782 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10783 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10784 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10785 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10786 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10788 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10789 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10791 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10792 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10793 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10794 ; GFX10-WGP-NEXT: s_endpgm
10795 ;
10796 ; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10797 ; GFX10-CU: ; %bb.0: ; %entry
10798 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10799 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10800 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10801 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10802 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10803 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10804 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10805 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10806 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10807 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10808 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10809 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10810 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10811 ; GFX10-CU-NEXT: s_endpgm
10812 ;
10813 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10814 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10815 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10816 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10817 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10818 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10819 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10820 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10821 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10822 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10823 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10824 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10825 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10826 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10827 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10828 ; SKIP-CACHE-INV-NEXT: s_endpgm
10829 ;
10830 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10831 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10832 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10833 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10834 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10835 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10836 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10837 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10838 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10839 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10840 ;
10841 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10842 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10843 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10844 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10845 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10846 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10847 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10848 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10849 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10850 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10851 ;
10852 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10853 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10854 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10855 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10856 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10857 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10858 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10859 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10860 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10861 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10862 ;
10863 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10864 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10865 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10866 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10867 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
10868 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
10869 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
10870 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10871 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
10872 ; GFX940-TGSPLIT-NEXT: s_endpgm
10873 ;
10874 ; GFX11-WGP-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10875 ; GFX11-WGP: ; %bb.0: ; %entry
10876 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10877 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10878 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10879 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10880 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10881 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10882 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
10883 ; GFX11-WGP-NEXT: s_endpgm
10884 ;
10885 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
10886 ; GFX11-CU: ; %bb.0: ; %entry
10887 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
10888 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10889 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
10890 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
10891 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
10892 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10893 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
10894 ; GFX11-CU-NEXT: s_endpgm
10895 ptr %out, i32 %in, i32 %old) {
10896 entry:
10897 %gep = getelementptr i32, ptr %out, i32 4
10898 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic seq_cst
10899 %val0 = extractvalue { i32, i1 } %val, 0
10900 store i32 %val0, ptr %out, align 4
10901 ret void
10902 }
10904 define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
10905 ; GFX7-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10906 ; GFX7: ; %bb.0: ; %entry
10907 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10908 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10909 ; GFX7-NEXT: s_add_u32 s4, s0, 16
10910 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
10911 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10912 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
10913 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10914 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
10915 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10916 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
10917 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
10918 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10919 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10920 ; GFX7-NEXT: s_endpgm
10921 ;
10922 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10923 ; GFX10-WGP: ; %bb.0: ; %entry
10924 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10925 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10926 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
10927 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
10928 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
10929 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
10930 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
10931 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
10932 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10933 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
10934 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
10935 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10936 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
10937 ; GFX10-WGP-NEXT: s_endpgm
10938 ;
10939 ; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10940 ; GFX10-CU: ; %bb.0: ; %entry
10941 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10942 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10943 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
10944 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
10945 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
10946 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
10947 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
10948 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
10949 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10950 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
10951 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
10952 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10953 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
10954 ; GFX10-CU-NEXT: s_endpgm
10955 ;
10956 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10957 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10958 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10959 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10960 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
10961 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
10962 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
10963 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
10964 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
10965 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
10966 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10967 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
10968 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
10969 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10970 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
10971 ; SKIP-CACHE-INV-NEXT: s_endpgm
10972 ;
10973 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10974 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10975 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10976 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10977 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10978 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10979 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10980 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10981 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
10982 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10983 ;
10984 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10985 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10986 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
10987 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10988 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
10989 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
10990 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
10991 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10992 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
10993 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10994 ;
10995 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
10996 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10997 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
10998 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10999 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11000 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11001 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11002 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11003 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11004 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11005 ;
11006 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
11007 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11008 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11009 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11010 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11011 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11012 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11013 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11014 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11015 ; GFX940-TGSPLIT-NEXT: s_endpgm
11016 ;
11017 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
11018 ; GFX11-WGP: ; %bb.0: ; %entry
11019 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11020 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11021 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11022 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11023 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11024 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11025 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11026 ; GFX11-WGP-NEXT: s_endpgm
11027 ;
11028 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
11029 ; GFX11-CU: ; %bb.0: ; %entry
11030 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11031 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11032 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11033 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11034 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11035 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11036 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11037 ; GFX11-CU-NEXT: s_endpgm
11038 ptr %out, i32 %in, i32 %old) {
11039 entry:
11040 %gep = getelementptr i32, ptr %out, i32 4
11041 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire seq_cst
11042 %val0 = extractvalue { i32, i1 } %val, 0
11043 store i32 %val0, ptr %out, align 4
11044 ret void
11045 }
11047 define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
11048 ; GFX7-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11049 ; GFX7: ; %bb.0: ; %entry
11050 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11051 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11052 ; GFX7-NEXT: s_add_u32 s4, s0, 16
11053 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
11054 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11055 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
11056 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11057 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
11058 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11059 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
11060 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
11061 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11062 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11063 ; GFX7-NEXT: s_endpgm
11064 ;
11065 ; GFX10-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11066 ; GFX10-WGP: ; %bb.0: ; %entry
11067 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11068 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11069 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
11070 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
11071 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11072 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
11073 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11074 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
11075 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11076 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
11077 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
11078 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11079 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11080 ; GFX10-WGP-NEXT: s_endpgm
11081 ;
11082 ; GFX10-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11083 ; GFX10-CU: ; %bb.0: ; %entry
11084 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11085 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11086 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
11087 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
11088 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11089 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
11090 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11091 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
11092 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11093 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
11094 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
11095 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11096 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11097 ; GFX10-CU-NEXT: s_endpgm
11098 ;
11099 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11100 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11101 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11102 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11103 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
11104 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
11105 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11106 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
11107 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
11108 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
11109 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11110 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11111 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11112 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11113 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11114 ; SKIP-CACHE-INV-NEXT: s_endpgm
11115 ;
11116 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11117 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11118 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11119 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11120 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11121 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11122 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11123 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11124 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11125 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11126 ;
11127 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11128 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11129 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11130 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11131 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11132 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11133 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11134 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11135 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11136 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11137 ;
11138 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11139 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11140 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11141 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11142 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11143 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11144 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11145 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11146 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11147 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11148 ;
11149 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11150 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11151 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11152 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11153 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11154 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11155 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11156 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11157 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11158 ; GFX940-TGSPLIT-NEXT: s_endpgm
11159 ;
11160 ; GFX11-WGP-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11161 ; GFX11-WGP: ; %bb.0: ; %entry
11162 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11163 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11164 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11165 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11166 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11167 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11168 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11169 ; GFX11-WGP-NEXT: s_endpgm
11170 ;
11171 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
11172 ; GFX11-CU: ; %bb.0: ; %entry
11173 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11174 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11175 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11176 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11177 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11178 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11179 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11180 ; GFX11-CU-NEXT: s_endpgm
11181 ptr %out, i32 %in, i32 %old) {
11182 entry:
11183 %gep = getelementptr i32, ptr %out, i32 4
11184 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release seq_cst
11185 %val0 = extractvalue { i32, i1 } %val, 0
11186 store i32 %val0, ptr %out, align 4
11187 ret void
11188 }
11190 define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
11191 ; GFX7-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11192 ; GFX7: ; %bb.0: ; %entry
11193 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11194 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11195 ; GFX7-NEXT: s_add_u32 s4, s0, 16
11196 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
11197 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11198 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
11199 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11200 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
11201 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11202 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
11203 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
11204 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11205 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11206 ; GFX7-NEXT: s_endpgm
11207 ;
11208 ; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11209 ; GFX10-WGP: ; %bb.0: ; %entry
11210 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11211 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11212 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
11213 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
11214 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11215 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
11216 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11217 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
11218 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11219 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
11220 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
11221 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11222 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11223 ; GFX10-WGP-NEXT: s_endpgm
11224 ;
11225 ; GFX10-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11226 ; GFX10-CU: ; %bb.0: ; %entry
11227 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11228 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11229 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
11230 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
11231 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11232 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
11233 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11234 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
11235 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11236 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
11237 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
11238 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11239 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11240 ; GFX10-CU-NEXT: s_endpgm
11241 ;
11242 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11243 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11244 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11245 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11246 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
11247 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
11248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11249 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
11250 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
11251 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
11252 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11253 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11254 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11255 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11256 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11257 ; SKIP-CACHE-INV-NEXT: s_endpgm
11258 ;
11259 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11260 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11261 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11262 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11263 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11264 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11265 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11266 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11267 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11268 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11269 ;
11270 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11271 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11272 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11273 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11274 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11275 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11276 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11277 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11278 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11279 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11280 ;
11281 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11282 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11283 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11284 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11285 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11286 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11287 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11288 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11289 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11290 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11291 ;
11292 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11293 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11294 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11295 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11296 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11297 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11298 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11299 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11300 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11301 ; GFX940-TGSPLIT-NEXT: s_endpgm
11302 ;
11303 ; GFX11-WGP-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11304 ; GFX11-WGP: ; %bb.0: ; %entry
11305 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11306 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11307 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11308 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11309 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11310 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11311 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11312 ; GFX11-WGP-NEXT: s_endpgm
11313 ;
11314 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
11315 ; GFX11-CU: ; %bb.0: ; %entry
11316 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11317 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11318 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11319 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11320 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11321 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11322 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11323 ; GFX11-CU-NEXT: s_endpgm
11324 ptr %out, i32 %in, i32 %old) {
11325 entry:
11326 %gep = getelementptr i32, ptr %out, i32 4
11327 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel seq_cst
11328 %val0 = extractvalue { i32, i1 } %val, 0
11329 store i32 %val0, ptr %out, align 4
11330 ret void
11331 }
11333 define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
11334 ; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11335 ; GFX7: ; %bb.0: ; %entry
11336 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11337 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11338 ; GFX7-NEXT: s_add_u32 s4, s0, 16
11339 ; GFX7-NEXT: s_addc_u32 s5, s1, 0
11340 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11341 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
11342 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11343 ; GFX7-NEXT: v_mov_b32_e32 v3, s3
11344 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11345 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
11346 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
11347 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11348 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11349 ; GFX7-NEXT: s_endpgm
11350 ;
11351 ; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11352 ; GFX10-WGP: ; %bb.0: ; %entry
11353 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11354 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11355 ; GFX10-WGP-NEXT: s_add_u32 s4, s0, 16
11356 ; GFX10-WGP-NEXT: s_addc_u32 s5, s1, 0
11357 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
11358 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
11359 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
11360 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s3
11361 ; GFX10-WGP-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11362 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
11363 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
11364 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11365 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
11366 ; GFX10-WGP-NEXT: s_endpgm
11367 ;
11368 ; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11369 ; GFX10-CU: ; %bb.0: ; %entry
11370 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11371 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11372 ; GFX10-CU-NEXT: s_add_u32 s4, s0, 16
11373 ; GFX10-CU-NEXT: s_addc_u32 s5, s1, 0
11374 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
11375 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
11376 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
11377 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s3
11378 ; GFX10-CU-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11379 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
11380 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
11381 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11382 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
11383 ; GFX10-CU-NEXT: s_endpgm
11384 ;
11385 ; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11386 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11387 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11388 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11389 ; SKIP-CACHE-INV-NEXT: s_add_u32 s4, s0, 16
11390 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s5, s1, 0
11391 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11392 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s2
11393 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s5
11394 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, s3
11395 ; SKIP-CACHE-INV-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
11396 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
11397 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
11398 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11399 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
11400 ; SKIP-CACHE-INV-NEXT: s_endpgm
11401 ;
11402 ; GFX90A-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11403 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11404 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11405 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11406 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11407 ; GFX90A-NOTTGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11408 ; GFX90A-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11409 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11410 ; GFX90A-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2
11411 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11412 ;
11413 ; GFX90A-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11414 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11415 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
11416 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11417 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
11418 ; GFX90A-TGSPLIT-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
11419 ; GFX90A-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
11420 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11421 ; GFX90A-TGSPLIT-NEXT: flat_store_dword v[0:1], v2
11422 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11423 ;
11424 ; GFX940-NOTTGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11425 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11426 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11427 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11428 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11429 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11430 ; GFX940-NOTTGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11431 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11432 ; GFX940-NOTTGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11433 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11434 ;
11435 ; GFX940-TGSPLIT-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11436 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11437 ; GFX940-TGSPLIT-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
11438 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11439 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
11440 ; GFX940-TGSPLIT-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
11441 ; GFX940-TGSPLIT-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 sc0
11442 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11443 ; GFX940-TGSPLIT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
11444 ; GFX940-TGSPLIT-NEXT: s_endpgm
11445 ;
11446 ; GFX11-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11447 ; GFX11-WGP: ; %bb.0: ; %entry
11448 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11449 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11450 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11451 ; GFX11-WGP-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11452 ; GFX11-WGP-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11453 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11454 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
11455 ; GFX11-WGP-NEXT: s_endpgm
11456 ;
11457 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
11458 ; GFX11-CU: ; %bb.0: ; %entry
11459 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
11460 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11461 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
11462 ; GFX11-CU-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
11463 ; GFX11-CU-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
11464 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11465 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
11466 ; GFX11-CU-NEXT: s_endpgm
11467 ptr %out, i32 %in, i32 %old) {
11468 entry:
11469 %gep = getelementptr i32, ptr %out, i32 4
11470 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
11471 %val0 = extractvalue { i32, i1 } %val, 0
11472 store i32 %val0, ptr %out, align 4
11473 ret void
11474 }