| blob | 30bf492071535247a08adc4c39d6d140f0103b05 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx600 < %s | FileCheck --check-prefixes=GFX6 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
12 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
13 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
14 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
16 define amdgpu_kernel void @global_workgroup_unordered_load(
17 ; GFX6-LABEL: global_workgroup_unordered_load:
18 ; GFX6: ; %bb.0: ; %entry
19 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
20 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
21 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
22 ; GFX6-NEXT: s_mov_b32 s6, s9
23 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
24 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
25 ; GFX6-NEXT: s_mov_b32 s13, -1
26 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
27 ; GFX6-NEXT: s_mov_b32 s9, s6
28 ; GFX6-NEXT: s_mov_b32 s10, s13
29 ; GFX6-NEXT: s_mov_b32 s11, s12
30 ; GFX6-NEXT: s_mov_b32 s14, s5
31 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
32 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
33 ; GFX6-NEXT: s_mov_b32 s5, s14
34 ; GFX6-NEXT: s_mov_b32 s6, s13
35 ; GFX6-NEXT: s_mov_b32 s7, s12
36 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
37 ; GFX6-NEXT: s_waitcnt vmcnt(0)
38 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
39 ; GFX6-NEXT: s_endpgm
40 ;
41 ; GFX7-LABEL: global_workgroup_unordered_load:
42 ; GFX7: ; %bb.0: ; %entry
43 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
44 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
45 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
46 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
47 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
48 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
49 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
50 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
51 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
52 ; GFX7-NEXT: s_waitcnt vmcnt(0)
53 ; GFX7-NEXT: flat_store_dword v[0:1], v2
54 ; GFX7-NEXT: s_endpgm
55 ;
56 ; GFX10-WGP-LABEL: global_workgroup_unordered_load:
57 ; GFX10-WGP: ; %bb.0: ; %entry
58 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
59 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
60 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
61 ; GFX10-WGP-NEXT: s_nop 0
62 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
63 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
64 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7]
65 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
66 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
67 ; GFX10-WGP-NEXT: s_endpgm
68 ;
69 ; GFX10-CU-LABEL: global_workgroup_unordered_load:
70 ; GFX10-CU: ; %bb.0: ; %entry
71 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
72 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
73 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
74 ; GFX10-CU-NEXT: s_nop 0
75 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
76 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
77 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
78 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
79 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
80 ; GFX10-CU-NEXT: s_endpgm
81 ;
82 ; SKIP-CACHE-INV-LABEL: global_workgroup_unordered_load:
83 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
84 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
85 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
86 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
87 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
88 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
89 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
90 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
91 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
92 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
93 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
94 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
95 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
96 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
97 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
98 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
99 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
100 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
101 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
102 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
103 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
104 ; SKIP-CACHE-INV-NEXT: s_endpgm
105 ;
106 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_unordered_load:
107 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
108 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
109 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
110 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
111 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
112 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
113 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
114 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
115 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
116 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
117 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
118 ;
119 ; GFX90A-TGSPLIT-LABEL: global_workgroup_unordered_load:
120 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
121 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
122 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
123 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
124 ; GFX90A-TGSPLIT-NEXT: s_nop 0
125 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
126 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
127 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
128 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
129 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
130 ; GFX90A-TGSPLIT-NEXT: s_endpgm
131 ;
132 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_unordered_load:
133 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
134 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
135 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
136 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
137 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
138 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
139 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
140 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3]
141 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
142 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
143 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
144 ;
145 ; GFX940-TGSPLIT-LABEL: global_workgroup_unordered_load:
146 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
147 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
148 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
149 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
150 ; GFX940-TGSPLIT-NEXT: s_nop 0
151 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
152 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
153 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3]
154 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
155 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
156 ; GFX940-TGSPLIT-NEXT: s_endpgm
157 ;
158 ; GFX11-WGP-LABEL: global_workgroup_unordered_load:
159 ; GFX11-WGP: ; %bb.0: ; %entry
160 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
161 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
162 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
163 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
164 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
165 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3]
166 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
167 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
168 ; GFX11-WGP-NEXT: s_endpgm
169 ;
170 ; GFX11-CU-LABEL: global_workgroup_unordered_load:
171 ; GFX11-CU: ; %bb.0: ; %entry
172 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
173 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
174 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
175 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
176 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
177 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
178 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
179 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
180 ; GFX11-CU-NEXT: s_endpgm
181 ;
182 ; GFX12-WGP-LABEL: global_workgroup_unordered_load:
183 ; GFX12-WGP: ; %bb.0: ; %entry
184 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
185 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
186 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
187 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
188 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
189 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3]
190 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
191 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
192 ; GFX12-WGP-NEXT: s_endpgm
193 ;
194 ; GFX12-CU-LABEL: global_workgroup_unordered_load:
195 ; GFX12-CU: ; %bb.0: ; %entry
196 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
197 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
198 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
199 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
200 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
201 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
202 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
203 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
204 ; GFX12-CU-NEXT: s_endpgm
205 ptr addrspace(1) %in, ptr addrspace(1) %out) {
206 entry:
207 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") unordered, align 4
208 store i32 %val, ptr addrspace(1) %out
209 ret void
210 }
212 define amdgpu_kernel void @global_workgroup_monotonic_load(
213 ; GFX6-LABEL: global_workgroup_monotonic_load:
214 ; GFX6: ; %bb.0: ; %entry
215 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
216 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
217 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
218 ; GFX6-NEXT: s_mov_b32 s6, s9
219 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
220 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
221 ; GFX6-NEXT: s_mov_b32 s13, -1
222 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
223 ; GFX6-NEXT: s_mov_b32 s9, s6
224 ; GFX6-NEXT: s_mov_b32 s10, s13
225 ; GFX6-NEXT: s_mov_b32 s11, s12
226 ; GFX6-NEXT: s_mov_b32 s14, s5
227 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
228 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
229 ; GFX6-NEXT: s_mov_b32 s5, s14
230 ; GFX6-NEXT: s_mov_b32 s6, s13
231 ; GFX6-NEXT: s_mov_b32 s7, s12
232 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
233 ; GFX6-NEXT: s_waitcnt vmcnt(0)
234 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
235 ; GFX6-NEXT: s_endpgm
236 ;
237 ; GFX7-LABEL: global_workgroup_monotonic_load:
238 ; GFX7: ; %bb.0: ; %entry
239 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
240 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
241 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
242 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
243 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
244 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
245 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
246 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
247 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
248 ; GFX7-NEXT: s_waitcnt vmcnt(0)
249 ; GFX7-NEXT: flat_store_dword v[0:1], v2
250 ; GFX7-NEXT: s_endpgm
251 ;
252 ; GFX10-WGP-LABEL: global_workgroup_monotonic_load:
253 ; GFX10-WGP: ; %bb.0: ; %entry
254 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
255 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
256 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
257 ; GFX10-WGP-NEXT: s_nop 0
258 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
259 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
260 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
261 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
262 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
263 ; GFX10-WGP-NEXT: s_endpgm
264 ;
265 ; GFX10-CU-LABEL: global_workgroup_monotonic_load:
266 ; GFX10-CU: ; %bb.0: ; %entry
267 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
268 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
269 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
270 ; GFX10-CU-NEXT: s_nop 0
271 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
272 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
273 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
274 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
275 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
276 ; GFX10-CU-NEXT: s_endpgm
277 ;
278 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_load:
279 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
280 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
281 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
282 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
283 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
284 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
285 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
286 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
287 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
288 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
289 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
290 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
291 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
292 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
293 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
294 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
296 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
297 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
298 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
299 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
300 ; SKIP-CACHE-INV-NEXT: s_endpgm
301 ;
302 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_load:
303 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
304 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
305 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
306 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
307 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
308 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
309 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
310 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
311 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
312 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
313 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
314 ;
315 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_load:
316 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
317 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
318 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
319 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
320 ; GFX90A-TGSPLIT-NEXT: s_nop 0
321 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
322 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
323 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
324 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
325 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
326 ; GFX90A-TGSPLIT-NEXT: s_endpgm
327 ;
328 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_load:
329 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
330 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
331 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
332 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
333 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
334 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
335 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
336 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
337 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
338 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
339 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
340 ;
341 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_load:
342 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
343 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
344 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
345 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
346 ; GFX940-TGSPLIT-NEXT: s_nop 0
347 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
348 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
349 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
350 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
351 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
352 ; GFX940-TGSPLIT-NEXT: s_endpgm
353 ;
354 ; GFX11-WGP-LABEL: global_workgroup_monotonic_load:
355 ; GFX11-WGP: ; %bb.0: ; %entry
356 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
357 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
358 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
359 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
360 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
361 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
362 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
363 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
364 ; GFX11-WGP-NEXT: s_endpgm
365 ;
366 ; GFX11-CU-LABEL: global_workgroup_monotonic_load:
367 ; GFX11-CU: ; %bb.0: ; %entry
368 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
369 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
370 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
371 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
372 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
373 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
374 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
375 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
376 ; GFX11-CU-NEXT: s_endpgm
377 ;
378 ; GFX12-WGP-LABEL: global_workgroup_monotonic_load:
379 ; GFX12-WGP: ; %bb.0: ; %entry
380 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
381 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
382 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
383 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
384 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
385 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
386 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
387 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
388 ; GFX12-WGP-NEXT: s_endpgm
389 ;
390 ; GFX12-CU-LABEL: global_workgroup_monotonic_load:
391 ; GFX12-CU: ; %bb.0: ; %entry
392 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
393 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
394 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
395 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
396 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
397 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
398 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
399 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
400 ; GFX12-CU-NEXT: s_endpgm
401 ptr addrspace(1) %in, ptr addrspace(1) %out) {
402 entry:
403 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") monotonic, align 4
404 store i32 %val, ptr addrspace(1) %out
405 ret void
406 }
408 define amdgpu_kernel void @global_workgroup_acquire_load(
409 ; GFX6-LABEL: global_workgroup_acquire_load:
410 ; GFX6: ; %bb.0: ; %entry
411 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
412 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
413 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
414 ; GFX6-NEXT: s_mov_b32 s6, s9
415 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
416 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
417 ; GFX6-NEXT: s_mov_b32 s13, -1
418 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
419 ; GFX6-NEXT: s_mov_b32 s9, s6
420 ; GFX6-NEXT: s_mov_b32 s10, s13
421 ; GFX6-NEXT: s_mov_b32 s11, s12
422 ; GFX6-NEXT: s_mov_b32 s14, s5
423 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
424 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
425 ; GFX6-NEXT: s_mov_b32 s5, s14
426 ; GFX6-NEXT: s_mov_b32 s6, s13
427 ; GFX6-NEXT: s_mov_b32 s7, s12
428 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
429 ; GFX6-NEXT: s_waitcnt vmcnt(0)
430 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
431 ; GFX6-NEXT: s_endpgm
432 ;
433 ; GFX7-LABEL: global_workgroup_acquire_load:
434 ; GFX7: ; %bb.0: ; %entry
435 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
436 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
437 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
438 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
439 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
440 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
441 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
442 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
443 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
444 ; GFX7-NEXT: s_waitcnt vmcnt(0)
445 ; GFX7-NEXT: flat_store_dword v[0:1], v2
446 ; GFX7-NEXT: s_endpgm
447 ;
448 ; GFX10-WGP-LABEL: global_workgroup_acquire_load:
449 ; GFX10-WGP: ; %bb.0: ; %entry
450 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
451 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
452 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
453 ; GFX10-WGP-NEXT: s_nop 0
454 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
455 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
456 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
457 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
458 ; GFX10-WGP-NEXT: buffer_gl0_inv
459 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
460 ; GFX10-WGP-NEXT: s_endpgm
461 ;
462 ; GFX10-CU-LABEL: global_workgroup_acquire_load:
463 ; GFX10-CU: ; %bb.0: ; %entry
464 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
465 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
466 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
467 ; GFX10-CU-NEXT: s_nop 0
468 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
469 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
470 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
471 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
472 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
473 ; GFX10-CU-NEXT: s_endpgm
474 ;
475 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_load:
476 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
477 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
478 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
479 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
480 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
481 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
482 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
483 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
484 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
485 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
486 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
487 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
488 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
489 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
490 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
491 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
492 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
493 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
494 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
495 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
496 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
497 ; SKIP-CACHE-INV-NEXT: s_endpgm
498 ;
499 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_load:
500 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
501 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
502 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
503 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
504 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
505 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
506 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
507 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
508 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
509 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
510 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
511 ;
512 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_load:
513 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
514 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
515 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
516 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
517 ; GFX90A-TGSPLIT-NEXT: s_nop 0
518 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
519 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
520 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
521 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
522 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
523 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
524 ; GFX90A-TGSPLIT-NEXT: s_endpgm
525 ;
526 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_load:
527 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
528 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
529 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
530 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
531 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
532 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
533 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
534 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
535 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
536 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
537 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
538 ;
539 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_load:
540 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
541 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
542 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
543 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
544 ; GFX940-TGSPLIT-NEXT: s_nop 0
545 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
546 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
547 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
548 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
549 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
550 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
551 ; GFX940-TGSPLIT-NEXT: s_endpgm
552 ;
553 ; GFX11-WGP-LABEL: global_workgroup_acquire_load:
554 ; GFX11-WGP: ; %bb.0: ; %entry
555 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
556 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
557 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
558 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
559 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
560 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
561 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
562 ; GFX11-WGP-NEXT: buffer_gl0_inv
563 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
564 ; GFX11-WGP-NEXT: s_endpgm
565 ;
566 ; GFX11-CU-LABEL: global_workgroup_acquire_load:
567 ; GFX11-CU: ; %bb.0: ; %entry
568 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
569 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
570 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
571 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
572 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
573 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
574 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
575 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
576 ; GFX11-CU-NEXT: s_endpgm
577 ;
578 ; GFX12-WGP-LABEL: global_workgroup_acquire_load:
579 ; GFX12-WGP: ; %bb.0: ; %entry
580 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
581 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
582 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
583 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
584 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
585 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
586 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
587 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
588 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
589 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
590 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
591 ; GFX12-WGP-NEXT: s_endpgm
592 ;
593 ; GFX12-CU-LABEL: global_workgroup_acquire_load:
594 ; GFX12-CU: ; %bb.0: ; %entry
595 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
596 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
597 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
598 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
599 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
600 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
601 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
602 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
603 ; GFX12-CU-NEXT: s_endpgm
604 ptr addrspace(1) %in, ptr addrspace(1) %out) {
605 entry:
606 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") acquire, align 4
607 store i32 %val, ptr addrspace(1) %out
608 ret void
609 }
611 define amdgpu_kernel void @global_workgroup_seq_cst_load(
612 ; GFX6-LABEL: global_workgroup_seq_cst_load:
613 ; GFX6: ; %bb.0: ; %entry
614 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
615 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
616 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
617 ; GFX6-NEXT: s_mov_b32 s6, s9
618 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
619 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
620 ; GFX6-NEXT: s_mov_b32 s13, -1
621 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
622 ; GFX6-NEXT: s_mov_b32 s9, s6
623 ; GFX6-NEXT: s_mov_b32 s10, s13
624 ; GFX6-NEXT: s_mov_b32 s11, s12
625 ; GFX6-NEXT: s_mov_b32 s14, s5
626 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
627 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
628 ; GFX6-NEXT: s_mov_b32 s5, s14
629 ; GFX6-NEXT: s_mov_b32 s6, s13
630 ; GFX6-NEXT: s_mov_b32 s7, s12
631 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
632 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
633 ; GFX6-NEXT: s_waitcnt vmcnt(0)
634 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
635 ; GFX6-NEXT: s_endpgm
636 ;
637 ; GFX7-LABEL: global_workgroup_seq_cst_load:
638 ; GFX7: ; %bb.0: ; %entry
639 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
640 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
641 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
642 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
643 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
644 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
645 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
646 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
647 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
648 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
649 ; GFX7-NEXT: s_waitcnt vmcnt(0)
650 ; GFX7-NEXT: flat_store_dword v[0:1], v2
651 ; GFX7-NEXT: s_endpgm
652 ;
653 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_load:
654 ; GFX10-WGP: ; %bb.0: ; %entry
655 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
656 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
657 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
658 ; GFX10-WGP-NEXT: s_nop 0
659 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
660 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
661 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
662 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
663 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
664 ; GFX10-WGP-NEXT: buffer_gl0_inv
665 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
666 ; GFX10-WGP-NEXT: s_endpgm
667 ;
668 ; GFX10-CU-LABEL: global_workgroup_seq_cst_load:
669 ; GFX10-CU: ; %bb.0: ; %entry
670 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
671 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
672 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
673 ; GFX10-CU-NEXT: s_nop 0
674 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
675 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
676 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
677 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
678 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
679 ; GFX10-CU-NEXT: s_endpgm
680 ;
681 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_load:
682 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
683 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
684 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
685 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
686 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
687 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
688 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
689 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
690 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
691 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
692 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
693 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
694 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
695 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
696 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
697 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
698 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
699 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
700 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
701 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
702 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
703 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
704 ; SKIP-CACHE-INV-NEXT: s_endpgm
705 ;
706 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_load:
707 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
708 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
709 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
710 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
711 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
712 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
713 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
714 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
715 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
716 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
717 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
718 ;
719 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_load:
720 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
721 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
722 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
723 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
724 ; GFX90A-TGSPLIT-NEXT: s_nop 0
725 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
726 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
727 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
728 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
729 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
730 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
731 ; GFX90A-TGSPLIT-NEXT: s_endpgm
732 ;
733 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_load:
734 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
735 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
736 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
737 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
738 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
739 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
740 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
741 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
742 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
743 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
744 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
745 ;
746 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_load:
747 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
748 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
749 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
750 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
751 ; GFX940-TGSPLIT-NEXT: s_nop 0
752 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
753 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
754 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
755 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
756 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
757 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
758 ; GFX940-TGSPLIT-NEXT: s_endpgm
759 ;
760 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_load:
761 ; GFX11-WGP: ; %bb.0: ; %entry
762 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
763 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
764 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
765 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
766 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
767 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
768 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
769 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
770 ; GFX11-WGP-NEXT: buffer_gl0_inv
771 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
772 ; GFX11-WGP-NEXT: s_endpgm
773 ;
774 ; GFX11-CU-LABEL: global_workgroup_seq_cst_load:
775 ; GFX11-CU: ; %bb.0: ; %entry
776 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
777 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
778 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
779 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
780 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
781 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
782 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
783 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
784 ; GFX11-CU-NEXT: s_endpgm
785 ;
786 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_load:
787 ; GFX12-WGP: ; %bb.0: ; %entry
788 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
789 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
790 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
791 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
792 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
793 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
794 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
795 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
796 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
797 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
798 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
799 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
800 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
801 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
802 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
803 ; GFX12-WGP-NEXT: s_endpgm
804 ;
805 ; GFX12-CU-LABEL: global_workgroup_seq_cst_load:
806 ; GFX12-CU: ; %bb.0: ; %entry
807 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
808 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
809 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
810 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
811 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
812 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
813 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
814 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
815 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
816 ; GFX12-CU-NEXT: s_endpgm
817 ptr addrspace(1) %in, ptr addrspace(1) %out) {
818 entry:
819 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") seq_cst, align 4
820 store i32 %val, ptr addrspace(1) %out
821 ret void
822 }
824 define amdgpu_kernel void @global_workgroup_unordered_store(
825 ; GFX6-LABEL: global_workgroup_unordered_store:
826 ; GFX6: ; %bb.0: ; %entry
827 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
828 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
829 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
830 ; GFX6-NEXT: s_mov_b32 s11, s5
831 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
832 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
833 ; GFX6-NEXT: s_mov_b32 s10, -1
834 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
835 ; GFX6-NEXT: s_mov_b32 s5, s11
836 ; GFX6-NEXT: s_mov_b32 s6, s10
837 ; GFX6-NEXT: s_mov_b32 s7, s9
838 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
839 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
840 ; GFX6-NEXT: s_endpgm
841 ;
842 ; GFX7-LABEL: global_workgroup_unordered_store:
843 ; GFX7: ; %bb.0: ; %entry
844 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
845 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
846 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
847 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
848 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
849 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
850 ; GFX7-NEXT: flat_store_dword v[0:1], v2
851 ; GFX7-NEXT: s_endpgm
852 ;
853 ; GFX10-WGP-LABEL: global_workgroup_unordered_store:
854 ; GFX10-WGP: ; %bb.0: ; %entry
855 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
856 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
857 ; GFX10-WGP-NEXT: s_nop 0
858 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
859 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
860 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
861 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
862 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
863 ; GFX10-WGP-NEXT: s_endpgm
864 ;
865 ; GFX10-CU-LABEL: global_workgroup_unordered_store:
866 ; GFX10-CU: ; %bb.0: ; %entry
867 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
868 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
869 ; GFX10-CU-NEXT: s_nop 0
870 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
871 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
872 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
873 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
874 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
875 ; GFX10-CU-NEXT: s_endpgm
876 ;
877 ; SKIP-CACHE-INV-LABEL: global_workgroup_unordered_store:
878 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
879 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
880 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
881 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
882 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
883 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
884 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
885 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
886 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
887 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
888 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
889 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
890 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
891 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
892 ; SKIP-CACHE-INV-NEXT: s_endpgm
893 ;
894 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_unordered_store:
895 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
896 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
897 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
898 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
899 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
900 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
901 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
902 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
903 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
904 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
905 ;
906 ; GFX90A-TGSPLIT-LABEL: global_workgroup_unordered_store:
907 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
908 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
909 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
910 ; GFX90A-TGSPLIT-NEXT: s_nop 0
911 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
912 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
913 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
914 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
915 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
916 ; GFX90A-TGSPLIT-NEXT: s_endpgm
917 ;
918 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_unordered_store:
919 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
920 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
921 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
922 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
923 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
924 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
925 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
926 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
927 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
928 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
929 ;
930 ; GFX940-TGSPLIT-LABEL: global_workgroup_unordered_store:
931 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
932 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
933 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
934 ; GFX940-TGSPLIT-NEXT: s_nop 0
935 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
936 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
937 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
938 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
939 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
940 ; GFX940-TGSPLIT-NEXT: s_endpgm
941 ;
942 ; GFX11-WGP-LABEL: global_workgroup_unordered_store:
943 ; GFX11-WGP: ; %bb.0: ; %entry
944 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
945 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
946 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
947 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
948 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
949 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
950 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
951 ; GFX11-WGP-NEXT: s_endpgm
952 ;
953 ; GFX11-CU-LABEL: global_workgroup_unordered_store:
954 ; GFX11-CU: ; %bb.0: ; %entry
955 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
956 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
957 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
958 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
959 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
960 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
961 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
962 ; GFX11-CU-NEXT: s_endpgm
963 ;
964 ; GFX12-WGP-LABEL: global_workgroup_unordered_store:
965 ; GFX12-WGP: ; %bb.0: ; %entry
966 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
967 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
968 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
969 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
970 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
971 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
972 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
973 ; GFX12-WGP-NEXT: s_endpgm
974 ;
975 ; GFX12-CU-LABEL: global_workgroup_unordered_store:
976 ; GFX12-CU: ; %bb.0: ; %entry
977 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
978 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
979 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
980 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
981 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
982 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
983 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
984 ; GFX12-CU-NEXT: s_endpgm
985 i32 %in, ptr addrspace(1) %out) {
986 entry:
987 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") unordered, align 4
988 ret void
989 }
991 define amdgpu_kernel void @global_workgroup_monotonic_store(
992 ; GFX6-LABEL: global_workgroup_monotonic_store:
993 ; GFX6: ; %bb.0: ; %entry
994 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
995 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
996 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
997 ; GFX6-NEXT: s_mov_b32 s11, s5
998 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
999 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1000 ; GFX6-NEXT: s_mov_b32 s10, -1
1001 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1002 ; GFX6-NEXT: s_mov_b32 s5, s11
1003 ; GFX6-NEXT: s_mov_b32 s6, s10
1004 ; GFX6-NEXT: s_mov_b32 s7, s9
1005 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1006 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
1007 ; GFX6-NEXT: s_endpgm
1008 ;
1009 ; GFX7-LABEL: global_workgroup_monotonic_store:
1010 ; GFX7: ; %bb.0: ; %entry
1011 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
1012 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
1013 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1014 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1015 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1016 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1017 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1018 ; GFX7-NEXT: s_endpgm
1019 ;
1020 ; GFX10-WGP-LABEL: global_workgroup_monotonic_store:
1021 ; GFX10-WGP: ; %bb.0: ; %entry
1022 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1023 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
1024 ; GFX10-WGP-NEXT: s_nop 0
1025 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1026 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1027 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1028 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1029 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
1030 ; GFX10-WGP-NEXT: s_endpgm
1031 ;
1032 ; GFX10-CU-LABEL: global_workgroup_monotonic_store:
1033 ; GFX10-CU: ; %bb.0: ; %entry
1034 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1035 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
1036 ; GFX10-CU-NEXT: s_nop 0
1037 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1038 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1039 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1040 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1041 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
1042 ; GFX10-CU-NEXT: s_endpgm
1043 ;
1044 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_store:
1045 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1046 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
1047 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
1048 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1049 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1050 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1051 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1052 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1053 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1054 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1055 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1056 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1057 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1058 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
1059 ; SKIP-CACHE-INV-NEXT: s_endpgm
1060 ;
1061 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_store:
1062 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1063 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1064 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1065 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1066 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1067 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1068 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1069 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1070 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1071 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1072 ;
1073 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_store:
1074 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1075 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1076 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1077 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1078 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1079 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1080 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1081 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1082 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1083 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1084 ;
1085 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_store:
1086 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1087 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1088 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1089 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1090 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1091 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1092 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1093 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1094 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1095 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1096 ;
1097 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_store:
1098 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1099 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1100 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1101 ; GFX940-TGSPLIT-NEXT: s_nop 0
1102 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1103 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1104 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1105 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1106 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1107 ; GFX940-TGSPLIT-NEXT: s_endpgm
1108 ;
1109 ; GFX11-WGP-LABEL: global_workgroup_monotonic_store:
1110 ; GFX11-WGP: ; %bb.0: ; %entry
1111 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1112 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1113 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1114 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1115 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1116 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1117 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
1118 ; GFX11-WGP-NEXT: s_endpgm
1119 ;
1120 ; GFX11-CU-LABEL: global_workgroup_monotonic_store:
1121 ; GFX11-CU: ; %bb.0: ; %entry
1122 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1123 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1124 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1125 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1126 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1127 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1128 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1129 ; GFX11-CU-NEXT: s_endpgm
1130 ;
1131 ; GFX12-WGP-LABEL: global_workgroup_monotonic_store:
1132 ; GFX12-WGP: ; %bb.0: ; %entry
1133 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1134 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1135 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1136 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1137 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1138 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
1139 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
1140 ; GFX12-WGP-NEXT: s_endpgm
1141 ;
1142 ; GFX12-CU-LABEL: global_workgroup_monotonic_store:
1143 ; GFX12-CU: ; %bb.0: ; %entry
1144 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1145 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1146 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1147 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1148 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1149 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
1150 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1151 ; GFX12-CU-NEXT: s_endpgm
1152 i32 %in, ptr addrspace(1) %out) {
1153 entry:
1154 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") monotonic, align 4
1155 ret void
1156 }
1158 define amdgpu_kernel void @global_workgroup_release_store(
1159 ; GFX6-LABEL: global_workgroup_release_store:
1160 ; GFX6: ; %bb.0: ; %entry
1161 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
1162 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
1163 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1164 ; GFX6-NEXT: s_mov_b32 s11, s5
1165 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
1166 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1167 ; GFX6-NEXT: s_mov_b32 s10, -1
1168 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1169 ; GFX6-NEXT: s_mov_b32 s5, s11
1170 ; GFX6-NEXT: s_mov_b32 s6, s10
1171 ; GFX6-NEXT: s_mov_b32 s7, s9
1172 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1173 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1174 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
1175 ; GFX6-NEXT: s_endpgm
1176 ;
1177 ; GFX7-LABEL: global_workgroup_release_store:
1178 ; GFX7: ; %bb.0: ; %entry
1179 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
1180 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
1181 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1182 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1183 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1184 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1185 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1186 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1187 ; GFX7-NEXT: s_endpgm
1188 ;
1189 ; GFX10-WGP-LABEL: global_workgroup_release_store:
1190 ; GFX10-WGP: ; %bb.0: ; %entry
1191 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1192 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
1193 ; GFX10-WGP-NEXT: s_nop 0
1194 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1195 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1196 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1197 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1198 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1199 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1200 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
1201 ; GFX10-WGP-NEXT: s_endpgm
1202 ;
1203 ; GFX10-CU-LABEL: global_workgroup_release_store:
1204 ; GFX10-CU: ; %bb.0: ; %entry
1205 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1206 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
1207 ; GFX10-CU-NEXT: s_nop 0
1208 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1209 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1210 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1211 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1212 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1213 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
1214 ; GFX10-CU-NEXT: s_endpgm
1215 ;
1216 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_store:
1217 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1218 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
1219 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
1220 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1221 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1222 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1223 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1224 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1225 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1226 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1227 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1228 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1229 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1230 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1231 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
1232 ; SKIP-CACHE-INV-NEXT: s_endpgm
1233 ;
1234 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_store:
1235 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1236 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1237 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1238 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1239 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1240 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1241 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1242 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1243 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1244 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1245 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1246 ;
1247 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_store:
1248 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1249 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1250 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1251 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1252 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1253 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1254 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1255 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1256 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1257 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1258 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1259 ;
1260 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_store:
1261 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1262 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1263 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1264 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1265 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1266 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1267 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1268 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1269 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1270 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1271 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1272 ;
1273 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_store:
1274 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1275 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1276 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1277 ; GFX940-TGSPLIT-NEXT: s_nop 0
1278 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1279 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1280 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1281 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1282 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1283 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1284 ; GFX940-TGSPLIT-NEXT: s_endpgm
1285 ;
1286 ; GFX11-WGP-LABEL: global_workgroup_release_store:
1287 ; GFX11-WGP: ; %bb.0: ; %entry
1288 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1289 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1290 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1291 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1292 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1293 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1294 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1295 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1296 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
1297 ; GFX11-WGP-NEXT: s_endpgm
1298 ;
1299 ; GFX11-CU-LABEL: global_workgroup_release_store:
1300 ; GFX11-CU: ; %bb.0: ; %entry
1301 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1302 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1303 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1304 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1305 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1306 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1307 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1308 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1309 ; GFX11-CU-NEXT: s_endpgm
1310 ;
1311 ; GFX12-WGP-LABEL: global_workgroup_release_store:
1312 ; GFX12-WGP: ; %bb.0: ; %entry
1313 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1314 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1315 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1316 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1317 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1318 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
1319 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
1320 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1321 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1322 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1323 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1324 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
1325 ; GFX12-WGP-NEXT: s_endpgm
1326 ;
1327 ; GFX12-CU-LABEL: global_workgroup_release_store:
1328 ; GFX12-CU: ; %bb.0: ; %entry
1329 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1330 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1331 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1332 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1333 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1334 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
1335 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1336 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1337 ; GFX12-CU-NEXT: s_endpgm
1338 i32 %in, ptr addrspace(1) %out) {
1339 entry:
1340 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") release, align 4
1341 ret void
1342 }
1344 define amdgpu_kernel void @global_workgroup_seq_cst_store(
1345 ; GFX6-LABEL: global_workgroup_seq_cst_store:
1346 ; GFX6: ; %bb.0: ; %entry
1347 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
1348 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
1349 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1350 ; GFX6-NEXT: s_mov_b32 s11, s5
1351 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
1352 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1353 ; GFX6-NEXT: s_mov_b32 s10, -1
1354 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1355 ; GFX6-NEXT: s_mov_b32 s5, s11
1356 ; GFX6-NEXT: s_mov_b32 s6, s10
1357 ; GFX6-NEXT: s_mov_b32 s7, s9
1358 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1359 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1360 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
1361 ; GFX6-NEXT: s_endpgm
1362 ;
1363 ; GFX7-LABEL: global_workgroup_seq_cst_store:
1364 ; GFX7: ; %bb.0: ; %entry
1365 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
1366 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
1367 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1368 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1369 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1370 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1371 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1372 ; GFX7-NEXT: flat_store_dword v[0:1], v2
1373 ; GFX7-NEXT: s_endpgm
1374 ;
1375 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_store:
1376 ; GFX10-WGP: ; %bb.0: ; %entry
1377 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
1378 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
1379 ; GFX10-WGP-NEXT: s_nop 0
1380 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1381 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1382 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1383 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1384 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1385 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1386 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
1387 ; GFX10-WGP-NEXT: s_endpgm
1388 ;
1389 ; GFX10-CU-LABEL: global_workgroup_seq_cst_store:
1390 ; GFX10-CU: ; %bb.0: ; %entry
1391 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
1392 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
1393 ; GFX10-CU-NEXT: s_nop 0
1394 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1395 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1396 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1397 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1398 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1399 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
1400 ; GFX10-CU-NEXT: s_endpgm
1401 ;
1402 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_store:
1403 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1404 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
1405 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
1406 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1407 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1408 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1409 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1410 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1411 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1412 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1413 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1414 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1415 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1416 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1417 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
1418 ; SKIP-CACHE-INV-NEXT: s_endpgm
1419 ;
1420 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_store:
1421 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1422 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1423 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1424 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1425 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1426 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1427 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1428 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1429 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1430 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1431 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1432 ;
1433 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_store:
1434 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1435 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1436 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1437 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1438 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1439 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1440 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1441 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1442 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1443 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1444 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1445 ;
1446 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_store:
1447 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1448 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1449 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1450 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1451 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1452 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1453 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1454 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1455 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1456 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1457 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1458 ;
1459 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_store:
1460 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1461 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1462 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1463 ; GFX940-TGSPLIT-NEXT: s_nop 0
1464 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1465 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1466 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1467 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1468 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1469 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1470 ; GFX940-TGSPLIT-NEXT: s_endpgm
1471 ;
1472 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_store:
1473 ; GFX11-WGP: ; %bb.0: ; %entry
1474 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1475 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1476 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1477 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1478 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1479 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1480 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1481 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1482 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
1483 ; GFX11-WGP-NEXT: s_endpgm
1484 ;
1485 ; GFX11-CU-LABEL: global_workgroup_seq_cst_store:
1486 ; GFX11-CU: ; %bb.0: ; %entry
1487 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1488 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1489 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1490 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1491 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1492 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1493 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1494 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1495 ; GFX11-CU-NEXT: s_endpgm
1496 ;
1497 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_store:
1498 ; GFX12-WGP: ; %bb.0: ; %entry
1499 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1500 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1501 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1502 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1503 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1504 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
1505 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
1506 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1507 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1508 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1509 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1510 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
1511 ; GFX12-WGP-NEXT: s_endpgm
1512 ;
1513 ; GFX12-CU-LABEL: global_workgroup_seq_cst_store:
1514 ; GFX12-CU: ; %bb.0: ; %entry
1515 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1516 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1517 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1518 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1519 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1520 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
1521 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1522 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1523 ; GFX12-CU-NEXT: s_endpgm
1524 i32 %in, ptr addrspace(1) %out) {
1525 entry:
1526 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") seq_cst, align 4
1527 ret void
1528 }
1530 define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
1531 ; GFX6-LABEL: global_workgroup_monotonic_atomicrmw:
1532 ; GFX6: ; %bb.0: ; %entry
1533 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1534 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
1535 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1536 ; GFX6-NEXT: s_mov_b32 s11, s5
1537 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
1538 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1539 ; GFX6-NEXT: s_mov_b32 s10, -1
1540 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1541 ; GFX6-NEXT: s_mov_b32 s5, s11
1542 ; GFX6-NEXT: s_mov_b32 s6, s10
1543 ; GFX6-NEXT: s_mov_b32 s7, s9
1544 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1545 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
1546 ; GFX6-NEXT: s_endpgm
1547 ;
1548 ; GFX7-LABEL: global_workgroup_monotonic_atomicrmw:
1549 ; GFX7: ; %bb.0: ; %entry
1550 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1551 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1552 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1553 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1554 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1555 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1556 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1557 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1558 ; GFX7-NEXT: s_endpgm
1559 ;
1560 ; GFX10-WGP-LABEL: global_workgroup_monotonic_atomicrmw:
1561 ; GFX10-WGP: ; %bb.0: ; %entry
1562 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1563 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1564 ; GFX10-WGP-NEXT: s_nop 0
1565 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
1566 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1567 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1568 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
1569 ; GFX10-WGP-NEXT: s_endpgm
1570 ;
1571 ; GFX10-CU-LABEL: global_workgroup_monotonic_atomicrmw:
1572 ; GFX10-CU: ; %bb.0: ; %entry
1573 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1574 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1575 ; GFX10-CU-NEXT: s_nop 0
1576 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
1577 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1578 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1579 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
1580 ; GFX10-CU-NEXT: s_endpgm
1581 ;
1582 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_atomicrmw:
1583 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1584 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1585 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
1586 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1587 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1588 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1589 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1590 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1591 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1592 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1593 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1594 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1595 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1596 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
1597 ; SKIP-CACHE-INV-NEXT: s_endpgm
1598 ;
1599 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_atomicrmw:
1600 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1601 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1602 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1603 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1604 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1605 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1606 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1607 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1608 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1609 ;
1610 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_atomicrmw:
1611 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1612 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1613 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1614 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1615 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1616 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1617 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1618 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1619 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1620 ;
1621 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_atomicrmw:
1622 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1623 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1624 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1625 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1626 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1627 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1628 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1629 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1630 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1631 ;
1632 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_atomicrmw:
1633 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1634 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1635 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1636 ; GFX940-TGSPLIT-NEXT: s_nop 0
1637 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1638 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1639 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1640 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1641 ; GFX940-TGSPLIT-NEXT: s_endpgm
1642 ;
1643 ; GFX11-WGP-LABEL: global_workgroup_monotonic_atomicrmw:
1644 ; GFX11-WGP: ; %bb.0: ; %entry
1645 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1646 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1647 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
1648 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1649 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1650 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1651 ; GFX11-WGP-NEXT: s_endpgm
1652 ;
1653 ; GFX11-CU-LABEL: global_workgroup_monotonic_atomicrmw:
1654 ; GFX11-CU: ; %bb.0: ; %entry
1655 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1656 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1657 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
1658 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1659 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1660 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1661 ; GFX11-CU-NEXT: s_endpgm
1662 ;
1663 ; GFX12-WGP-LABEL: global_workgroup_monotonic_atomicrmw:
1664 ; GFX12-WGP: ; %bb.0: ; %entry
1665 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1666 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1667 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
1668 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1669 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
1670 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
1671 ; GFX12-WGP-NEXT: s_endpgm
1672 ;
1673 ; GFX12-CU-LABEL: global_workgroup_monotonic_atomicrmw:
1674 ; GFX12-CU: ; %bb.0: ; %entry
1675 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1676 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1677 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
1678 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1679 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
1680 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1681 ; GFX12-CU-NEXT: s_endpgm
1682 ptr addrspace(1) %out, i32 %in) {
1683 entry:
1684 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") monotonic
1685 ret void
1686 }
1688 define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
1689 ; GFX6-LABEL: global_workgroup_acquire_atomicrmw:
1690 ; GFX6: ; %bb.0: ; %entry
1691 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1692 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
1693 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1694 ; GFX6-NEXT: s_mov_b32 s11, s5
1695 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
1696 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1697 ; GFX6-NEXT: s_mov_b32 s10, -1
1698 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1699 ; GFX6-NEXT: s_mov_b32 s5, s11
1700 ; GFX6-NEXT: s_mov_b32 s6, s10
1701 ; GFX6-NEXT: s_mov_b32 s7, s9
1702 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1703 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
1704 ; GFX6-NEXT: s_endpgm
1705 ;
1706 ; GFX7-LABEL: global_workgroup_acquire_atomicrmw:
1707 ; GFX7: ; %bb.0: ; %entry
1708 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1709 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1710 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1711 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1712 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1713 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1714 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1715 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1716 ; GFX7-NEXT: s_endpgm
1717 ;
1718 ; GFX10-WGP-LABEL: global_workgroup_acquire_atomicrmw:
1719 ; GFX10-WGP: ; %bb.0: ; %entry
1720 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1721 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1722 ; GFX10-WGP-NEXT: s_nop 0
1723 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
1724 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1725 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1726 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
1727 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1728 ; GFX10-WGP-NEXT: buffer_gl0_inv
1729 ; GFX10-WGP-NEXT: s_endpgm
1730 ;
1731 ; GFX10-CU-LABEL: global_workgroup_acquire_atomicrmw:
1732 ; GFX10-CU: ; %bb.0: ; %entry
1733 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1734 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1735 ; GFX10-CU-NEXT: s_nop 0
1736 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
1737 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1738 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1739 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
1740 ; GFX10-CU-NEXT: s_endpgm
1741 ;
1742 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_atomicrmw:
1743 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1744 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1745 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
1746 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1747 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1748 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1749 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1750 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1751 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1752 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1753 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1754 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1755 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1756 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
1757 ; SKIP-CACHE-INV-NEXT: s_endpgm
1758 ;
1759 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_atomicrmw:
1760 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1761 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1762 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1763 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1764 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1765 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1766 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1767 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1768 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1769 ;
1770 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_atomicrmw:
1771 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1772 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1773 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1774 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1775 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1776 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1777 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1778 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1779 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1780 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
1781 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1782 ;
1783 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_atomicrmw:
1784 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1785 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1786 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1787 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1788 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1789 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1790 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1791 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1792 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1793 ;
1794 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_atomicrmw:
1795 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1796 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1797 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1798 ; GFX940-TGSPLIT-NEXT: s_nop 0
1799 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1800 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1801 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1802 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1803 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1804 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
1805 ; GFX940-TGSPLIT-NEXT: s_endpgm
1806 ;
1807 ; GFX11-WGP-LABEL: global_workgroup_acquire_atomicrmw:
1808 ; GFX11-WGP: ; %bb.0: ; %entry
1809 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1810 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1811 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
1812 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1813 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1814 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1815 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1816 ; GFX11-WGP-NEXT: buffer_gl0_inv
1817 ; GFX11-WGP-NEXT: s_endpgm
1818 ;
1819 ; GFX11-CU-LABEL: global_workgroup_acquire_atomicrmw:
1820 ; GFX11-CU: ; %bb.0: ; %entry
1821 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1822 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1823 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
1824 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1825 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1826 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1827 ; GFX11-CU-NEXT: s_endpgm
1828 ;
1829 ; GFX12-WGP-LABEL: global_workgroup_acquire_atomicrmw:
1830 ; GFX12-WGP: ; %bb.0: ; %entry
1831 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1832 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1833 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
1834 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1835 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
1836 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
1837 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1838 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
1839 ; GFX12-WGP-NEXT: s_endpgm
1840 ;
1841 ; GFX12-CU-LABEL: global_workgroup_acquire_atomicrmw:
1842 ; GFX12-CU: ; %bb.0: ; %entry
1843 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1844 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1845 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
1846 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1847 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
1848 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1849 ; GFX12-CU-NEXT: s_endpgm
1850 ptr addrspace(1) %out, i32 %in) {
1851 entry:
1852 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
1853 ret void
1854 }
1856 define amdgpu_kernel void @global_workgroup_release_atomicrmw(
1857 ; GFX6-LABEL: global_workgroup_release_atomicrmw:
1858 ; GFX6: ; %bb.0: ; %entry
1859 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1860 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
1861 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1862 ; GFX6-NEXT: s_mov_b32 s11, s5
1863 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
1864 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
1865 ; GFX6-NEXT: s_mov_b32 s10, -1
1866 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
1867 ; GFX6-NEXT: s_mov_b32 s5, s11
1868 ; GFX6-NEXT: s_mov_b32 s6, s10
1869 ; GFX6-NEXT: s_mov_b32 s7, s9
1870 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
1871 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
1872 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
1873 ; GFX6-NEXT: s_endpgm
1874 ;
1875 ; GFX7-LABEL: global_workgroup_release_atomicrmw:
1876 ; GFX7: ; %bb.0: ; %entry
1877 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
1878 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
1879 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
1880 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1881 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
1882 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
1883 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
1884 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1885 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
1886 ; GFX7-NEXT: s_endpgm
1887 ;
1888 ; GFX10-WGP-LABEL: global_workgroup_release_atomicrmw:
1889 ; GFX10-WGP: ; %bb.0: ; %entry
1890 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
1891 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1892 ; GFX10-WGP-NEXT: s_nop 0
1893 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
1894 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
1895 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
1896 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1897 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1898 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
1899 ; GFX10-WGP-NEXT: s_endpgm
1900 ;
1901 ; GFX10-CU-LABEL: global_workgroup_release_atomicrmw:
1902 ; GFX10-CU: ; %bb.0: ; %entry
1903 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
1904 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1905 ; GFX10-CU-NEXT: s_nop 0
1906 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
1907 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1908 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
1909 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
1910 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
1911 ; GFX10-CU-NEXT: s_endpgm
1912 ;
1913 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_atomicrmw:
1914 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
1915 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1916 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
1917 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1918 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
1919 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
1920 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
1921 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
1922 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
1923 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
1924 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
1925 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
1926 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
1927 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
1928 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
1929 ; SKIP-CACHE-INV-NEXT: s_endpgm
1930 ;
1931 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_atomicrmw:
1932 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1933 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1934 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1935 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1936 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1937 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1938 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1939 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1940 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1941 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1942 ;
1943 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_atomicrmw:
1944 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1945 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1946 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
1947 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1948 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
1949 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1950 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1951 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1952 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
1953 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1954 ;
1955 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_atomicrmw:
1956 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1957 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1958 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1959 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1960 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1961 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1962 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1963 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1964 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1965 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1966 ;
1967 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_atomicrmw:
1968 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1969 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1970 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
1971 ; GFX940-TGSPLIT-NEXT: s_nop 0
1972 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
1973 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1974 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
1975 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1976 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
1977 ; GFX940-TGSPLIT-NEXT: s_endpgm
1978 ;
1979 ; GFX11-WGP-LABEL: global_workgroup_release_atomicrmw:
1980 ; GFX11-WGP: ; %bb.0: ; %entry
1981 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1982 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1983 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
1984 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1985 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
1986 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1987 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
1988 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
1989 ; GFX11-WGP-NEXT: s_endpgm
1990 ;
1991 ; GFX11-CU-LABEL: global_workgroup_release_atomicrmw:
1992 ; GFX11-CU: ; %bb.0: ; %entry
1993 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1994 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
1995 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
1996 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1997 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
1998 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1999 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2000 ; GFX11-CU-NEXT: s_endpgm
2001 ;
2002 ; GFX12-WGP-LABEL: global_workgroup_release_atomicrmw:
2003 ; GFX12-WGP: ; %bb.0: ; %entry
2004 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2005 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2006 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2007 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2008 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2009 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2010 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2011 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2012 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2013 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2014 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
2015 ; GFX12-WGP-NEXT: s_endpgm
2016 ;
2017 ; GFX12-CU-LABEL: global_workgroup_release_atomicrmw:
2018 ; GFX12-CU: ; %bb.0: ; %entry
2019 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
2020 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2021 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2022 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2023 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
2024 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2025 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2026 ; GFX12-CU-NEXT: s_endpgm
2027 ptr addrspace(1) %out, i32 %in) {
2028 entry:
2029 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") release
2030 ret void
2031 }
2033 define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
2034 ; GFX6-LABEL: global_workgroup_acq_rel_atomicrmw:
2035 ; GFX6: ; %bb.0: ; %entry
2036 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2037 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
2038 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2039 ; GFX6-NEXT: s_mov_b32 s11, s5
2040 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
2041 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
2042 ; GFX6-NEXT: s_mov_b32 s10, -1
2043 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
2044 ; GFX6-NEXT: s_mov_b32 s5, s11
2045 ; GFX6-NEXT: s_mov_b32 s6, s10
2046 ; GFX6-NEXT: s_mov_b32 s7, s9
2047 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
2048 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2049 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
2050 ; GFX6-NEXT: s_endpgm
2051 ;
2052 ; GFX7-LABEL: global_workgroup_acq_rel_atomicrmw:
2053 ; GFX7: ; %bb.0: ; %entry
2054 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
2055 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2056 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
2057 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2058 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2059 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2060 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2061 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2062 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
2063 ; GFX7-NEXT: s_endpgm
2064 ;
2065 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_atomicrmw:
2066 ; GFX10-WGP: ; %bb.0: ; %entry
2067 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
2068 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2069 ; GFX10-WGP-NEXT: s_nop 0
2070 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2071 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2072 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
2073 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2074 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2075 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
2076 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2077 ; GFX10-WGP-NEXT: buffer_gl0_inv
2078 ; GFX10-WGP-NEXT: s_endpgm
2079 ;
2080 ; GFX10-CU-LABEL: global_workgroup_acq_rel_atomicrmw:
2081 ; GFX10-CU: ; %bb.0: ; %entry
2082 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
2083 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2084 ; GFX10-CU-NEXT: s_nop 0
2085 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2086 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2087 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
2088 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2089 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
2090 ; GFX10-CU-NEXT: s_endpgm
2091 ;
2092 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_atomicrmw:
2093 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2094 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2095 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
2096 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2097 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
2098 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
2099 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
2100 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
2101 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
2102 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
2103 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
2104 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
2105 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
2106 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2107 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
2108 ; SKIP-CACHE-INV-NEXT: s_endpgm
2109 ;
2110 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_atomicrmw:
2111 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2112 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2113 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2114 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2115 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2116 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2117 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2118 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2119 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
2120 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2121 ;
2122 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_atomicrmw:
2123 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2124 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2125 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2126 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2127 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2128 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2129 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2130 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2131 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
2132 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2133 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2134 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2135 ;
2136 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_atomicrmw:
2137 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2138 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2139 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2140 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2141 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2142 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2143 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2144 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2145 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
2146 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2147 ;
2148 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_atomicrmw:
2149 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2150 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2151 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2152 ; GFX940-TGSPLIT-NEXT: s_nop 0
2153 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2154 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2155 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2156 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2157 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
2158 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2159 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2160 ; GFX940-TGSPLIT-NEXT: s_endpgm
2161 ;
2162 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_atomicrmw:
2163 ; GFX11-WGP: ; %bb.0: ; %entry
2164 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
2165 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2166 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2167 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2168 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
2169 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2170 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2171 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2172 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2173 ; GFX11-WGP-NEXT: buffer_gl0_inv
2174 ; GFX11-WGP-NEXT: s_endpgm
2175 ;
2176 ; GFX11-CU-LABEL: global_workgroup_acq_rel_atomicrmw:
2177 ; GFX11-CU: ; %bb.0: ; %entry
2178 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
2179 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2180 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2181 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2182 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
2183 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2184 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2185 ; GFX11-CU-NEXT: s_endpgm
2186 ;
2187 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_atomicrmw:
2188 ; GFX12-WGP: ; %bb.0: ; %entry
2189 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2190 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2191 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2192 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2193 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2194 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2195 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2196 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2197 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2198 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2199 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
2200 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2201 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2202 ; GFX12-WGP-NEXT: s_endpgm
2203 ;
2204 ; GFX12-CU-LABEL: global_workgroup_acq_rel_atomicrmw:
2205 ; GFX12-CU: ; %bb.0: ; %entry
2206 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
2207 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2208 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2209 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2210 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
2211 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2212 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2213 ; GFX12-CU-NEXT: s_endpgm
2214 ptr addrspace(1) %out, i32 %in) {
2215 entry:
2216 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
2217 ret void
2218 }
2220 define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
2221 ; GFX6-LABEL: global_workgroup_seq_cst_atomicrmw:
2222 ; GFX6: ; %bb.0: ; %entry
2223 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2224 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
2225 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2226 ; GFX6-NEXT: s_mov_b32 s11, s5
2227 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
2228 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
2229 ; GFX6-NEXT: s_mov_b32 s10, -1
2230 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
2231 ; GFX6-NEXT: s_mov_b32 s5, s11
2232 ; GFX6-NEXT: s_mov_b32 s6, s10
2233 ; GFX6-NEXT: s_mov_b32 s7, s9
2234 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
2235 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2236 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
2237 ; GFX6-NEXT: s_endpgm
2238 ;
2239 ; GFX7-LABEL: global_workgroup_seq_cst_atomicrmw:
2240 ; GFX7: ; %bb.0: ; %entry
2241 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
2242 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
2243 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
2244 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2245 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
2246 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
2247 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
2248 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2249 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
2250 ; GFX7-NEXT: s_endpgm
2251 ;
2252 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_atomicrmw:
2253 ; GFX10-WGP: ; %bb.0: ; %entry
2254 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
2255 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2256 ; GFX10-WGP-NEXT: s_nop 0
2257 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2258 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2259 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
2260 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2261 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2262 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
2263 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2264 ; GFX10-WGP-NEXT: buffer_gl0_inv
2265 ; GFX10-WGP-NEXT: s_endpgm
2266 ;
2267 ; GFX10-CU-LABEL: global_workgroup_seq_cst_atomicrmw:
2268 ; GFX10-CU: ; %bb.0: ; %entry
2269 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
2270 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2271 ; GFX10-CU-NEXT: s_nop 0
2272 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2273 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2274 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
2275 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2276 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
2277 ; GFX10-CU-NEXT: s_endpgm
2278 ;
2279 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_atomicrmw:
2280 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2281 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2282 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
2283 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2284 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
2285 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
2286 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
2287 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
2288 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
2289 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
2290 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
2291 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
2292 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
2293 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2294 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
2295 ; SKIP-CACHE-INV-NEXT: s_endpgm
2296 ;
2297 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_atomicrmw:
2298 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2299 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2300 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2301 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2302 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2303 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2304 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2305 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2306 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
2307 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2308 ;
2309 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_atomicrmw:
2310 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2311 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2312 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2313 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2314 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2315 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2316 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2317 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2318 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
2319 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2320 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2321 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2322 ;
2323 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_atomicrmw:
2324 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2325 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2326 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2327 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2328 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2329 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2330 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2331 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2332 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
2333 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2334 ;
2335 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_atomicrmw:
2336 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2337 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2338 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2339 ; GFX940-TGSPLIT-NEXT: s_nop 0
2340 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2341 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2342 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2343 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2344 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
2345 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2346 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2347 ; GFX940-TGSPLIT-NEXT: s_endpgm
2348 ;
2349 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_atomicrmw:
2350 ; GFX11-WGP: ; %bb.0: ; %entry
2351 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
2352 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2353 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2354 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2355 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
2356 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2357 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2358 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2359 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2360 ; GFX11-WGP-NEXT: buffer_gl0_inv
2361 ; GFX11-WGP-NEXT: s_endpgm
2362 ;
2363 ; GFX11-CU-LABEL: global_workgroup_seq_cst_atomicrmw:
2364 ; GFX11-CU: ; %bb.0: ; %entry
2365 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
2366 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2367 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2368 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2369 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
2370 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2371 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2372 ; GFX11-CU-NEXT: s_endpgm
2373 ;
2374 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_atomicrmw:
2375 ; GFX12-WGP: ; %bb.0: ; %entry
2376 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2377 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2378 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2379 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2380 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2381 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2382 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2383 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2384 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2385 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2386 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
2387 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2388 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2389 ; GFX12-WGP-NEXT: s_endpgm
2390 ;
2391 ; GFX12-CU-LABEL: global_workgroup_seq_cst_atomicrmw:
2392 ; GFX12-CU: ; %bb.0: ; %entry
2393 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
2394 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2395 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2396 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2397 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
2398 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2399 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
2400 ; GFX12-CU-NEXT: s_endpgm
2401 ptr addrspace(1) %out, i32 %in) {
2402 entry:
2403 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
2404 ret void
2405 }
2407 define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
2408 ; GFX6-LABEL: global_workgroup_acquire_ret_atomicrmw:
2409 ; GFX6: ; %bb.0: ; %entry
2410 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2411 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
2412 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2413 ; GFX6-NEXT: s_mov_b32 s11, s5
2414 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
2415 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
2416 ; GFX6-NEXT: s_mov_b32 s10, -1
2417 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
2418 ; GFX6-NEXT: s_mov_b32 s5, s11
2419 ; GFX6-NEXT: s_mov_b32 s6, s10
2420 ; GFX6-NEXT: s_mov_b32 s7, s9
2421 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
2422 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
2423 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2424 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
2425 ; GFX6-NEXT: s_endpgm
2426 ;
2427 ; GFX7-LABEL: global_workgroup_acquire_ret_atomicrmw:
2428 ; GFX7: ; %bb.0: ; %entry
2429 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2430 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2431 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2432 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2433 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2434 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2435 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2436 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2437 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2438 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2439 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2440 ; GFX7-NEXT: s_endpgm
2441 ;
2442 ; GFX10-WGP-LABEL: global_workgroup_acquire_ret_atomicrmw:
2443 ; GFX10-WGP: ; %bb.0: ; %entry
2444 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
2445 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2446 ; GFX10-WGP-NEXT: s_nop 0
2447 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2448 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2449 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
2450 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2451 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
2452 ; GFX10-WGP-NEXT: buffer_gl0_inv
2453 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
2454 ; GFX10-WGP-NEXT: s_endpgm
2455 ;
2456 ; GFX10-CU-LABEL: global_workgroup_acquire_ret_atomicrmw:
2457 ; GFX10-CU: ; %bb.0: ; %entry
2458 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
2459 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2460 ; GFX10-CU-NEXT: s_nop 0
2461 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2462 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2463 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
2464 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2465 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2466 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
2467 ; GFX10-CU-NEXT: s_endpgm
2468 ;
2469 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_ret_atomicrmw:
2470 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2471 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2472 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
2473 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2474 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
2475 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
2476 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
2477 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
2478 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
2479 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
2480 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
2481 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
2482 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
2483 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
2484 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2485 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
2486 ; SKIP-CACHE-INV-NEXT: s_endpgm
2487 ;
2488 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_ret_atomicrmw:
2489 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2490 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2491 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2492 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2493 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2494 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2495 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2496 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2497 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2498 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2499 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2500 ;
2501 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_ret_atomicrmw:
2502 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2503 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2504 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2505 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2506 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2507 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2508 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2509 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2510 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2511 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2512 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2513 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2514 ;
2515 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_ret_atomicrmw:
2516 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2517 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2518 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2519 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2520 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2521 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2522 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2523 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2524 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2525 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2526 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2527 ;
2528 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_ret_atomicrmw:
2529 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2530 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2531 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2532 ; GFX940-TGSPLIT-NEXT: s_nop 0
2533 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2534 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2535 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2536 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2537 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2538 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2539 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2540 ; GFX940-TGSPLIT-NEXT: s_endpgm
2541 ;
2542 ; GFX11-WGP-LABEL: global_workgroup_acquire_ret_atomicrmw:
2543 ; GFX11-WGP: ; %bb.0: ; %entry
2544 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
2545 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2546 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2547 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2548 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
2549 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2550 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
2551 ; GFX11-WGP-NEXT: buffer_gl0_inv
2552 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
2553 ; GFX11-WGP-NEXT: s_endpgm
2554 ;
2555 ; GFX11-CU-LABEL: global_workgroup_acquire_ret_atomicrmw:
2556 ; GFX11-CU: ; %bb.0: ; %entry
2557 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
2558 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2559 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2560 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2561 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
2562 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2563 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2564 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
2565 ; GFX11-CU-NEXT: s_endpgm
2566 ;
2567 ; GFX12-WGP-LABEL: global_workgroup_acquire_ret_atomicrmw:
2568 ; GFX12-WGP: ; %bb.0: ; %entry
2569 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2570 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2571 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2572 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2573 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2574 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
2575 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2576 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2577 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
2578 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2579 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
2580 ; GFX12-WGP-NEXT: s_endpgm
2581 ;
2582 ; GFX12-CU-LABEL: global_workgroup_acquire_ret_atomicrmw:
2583 ; GFX12-CU: ; %bb.0: ; %entry
2584 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
2585 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2586 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2587 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2588 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
2589 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
2590 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
2591 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
2592 ; GFX12-CU-NEXT: s_endpgm
2593 ptr addrspace(1) %out, i32 %in) {
2594 entry:
2595 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
2596 store i32 %val, ptr addrspace(1) %out, align 4
2597 ret void
2598 }
2600 define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
2601 ; GFX6-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2602 ; GFX6: ; %bb.0: ; %entry
2603 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2604 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
2605 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2606 ; GFX6-NEXT: s_mov_b32 s11, s5
2607 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
2608 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
2609 ; GFX6-NEXT: s_mov_b32 s10, -1
2610 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
2611 ; GFX6-NEXT: s_mov_b32 s5, s11
2612 ; GFX6-NEXT: s_mov_b32 s6, s10
2613 ; GFX6-NEXT: s_mov_b32 s7, s9
2614 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
2615 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2616 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
2617 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2618 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
2619 ; GFX6-NEXT: s_endpgm
2620 ;
2621 ; GFX7-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2622 ; GFX7: ; %bb.0: ; %entry
2623 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2624 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2625 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2626 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2627 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2628 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2629 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2630 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2631 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2632 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2633 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2634 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2635 ; GFX7-NEXT: s_endpgm
2636 ;
2637 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2638 ; GFX10-WGP: ; %bb.0: ; %entry
2639 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
2640 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2641 ; GFX10-WGP-NEXT: s_nop 0
2642 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2643 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2644 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
2645 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2646 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2647 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2648 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
2649 ; GFX10-WGP-NEXT: buffer_gl0_inv
2650 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
2651 ; GFX10-WGP-NEXT: s_endpgm
2652 ;
2653 ; GFX10-CU-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2654 ; GFX10-CU: ; %bb.0: ; %entry
2655 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
2656 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2657 ; GFX10-CU-NEXT: s_nop 0
2658 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2659 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2660 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
2661 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2662 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2663 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2664 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
2665 ; GFX10-CU-NEXT: s_endpgm
2666 ;
2667 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2668 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2669 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2670 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
2671 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2672 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
2673 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
2674 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
2675 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
2676 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
2677 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
2678 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
2679 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
2680 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
2681 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2682 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
2683 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2684 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
2685 ; SKIP-CACHE-INV-NEXT: s_endpgm
2686 ;
2687 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2688 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2689 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2690 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2691 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2692 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2693 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2694 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2695 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2696 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2697 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2698 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2699 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2700 ;
2701 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2702 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2703 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2704 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2705 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2706 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2707 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2708 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2709 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2710 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2711 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2712 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2713 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2714 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2715 ;
2716 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2717 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2718 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2719 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2720 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2721 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2722 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2723 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2724 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2725 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2726 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2727 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2728 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2729 ;
2730 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2731 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2732 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2733 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2734 ; GFX940-TGSPLIT-NEXT: s_nop 0
2735 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2736 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2737 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2738 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2739 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2740 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2741 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2742 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2743 ; GFX940-TGSPLIT-NEXT: s_endpgm
2744 ;
2745 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2746 ; GFX11-WGP: ; %bb.0: ; %entry
2747 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
2748 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2749 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2750 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2751 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
2752 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2753 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2754 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2755 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
2756 ; GFX11-WGP-NEXT: buffer_gl0_inv
2757 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
2758 ; GFX11-WGP-NEXT: s_endpgm
2759 ;
2760 ; GFX11-CU-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2761 ; GFX11-CU: ; %bb.0: ; %entry
2762 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
2763 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2764 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2765 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2766 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
2767 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2768 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2769 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2770 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
2771 ; GFX11-CU-NEXT: s_endpgm
2772 ;
2773 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2774 ; GFX12-WGP: ; %bb.0: ; %entry
2775 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2776 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2777 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2778 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2779 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2780 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2781 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2782 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2783 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2784 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2785 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
2786 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2787 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2788 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
2789 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
2790 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
2791 ; GFX12-WGP-NEXT: s_endpgm
2792 ;
2793 ; GFX12-CU-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
2794 ; GFX12-CU: ; %bb.0: ; %entry
2795 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
2796 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2797 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2798 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
2799 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
2800 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
2801 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
2802 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
2803 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
2804 ; GFX12-CU-NEXT: s_endpgm
2805 ptr addrspace(1) %out, i32 %in) {
2806 entry:
2807 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
2808 store i32 %val, ptr addrspace(1) %out, align 4
2809 ret void
2810 }
2812 define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
2813 ; GFX6-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2814 ; GFX6: ; %bb.0: ; %entry
2815 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2816 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
2817 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2818 ; GFX6-NEXT: s_mov_b32 s11, s5
2819 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
2820 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
2821 ; GFX6-NEXT: s_mov_b32 s10, -1
2822 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
2823 ; GFX6-NEXT: s_mov_b32 s5, s11
2824 ; GFX6-NEXT: s_mov_b32 s6, s10
2825 ; GFX6-NEXT: s_mov_b32 s7, s9
2826 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
2827 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
2828 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
2829 ; GFX6-NEXT: s_waitcnt vmcnt(0)
2830 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
2831 ; GFX6-NEXT: s_endpgm
2832 ;
2833 ; GFX7-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2834 ; GFX7: ; %bb.0: ; %entry
2835 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2836 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
2837 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2838 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2839 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2840 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
2841 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2842 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
2843 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
2844 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
2845 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2846 ; GFX7-NEXT: flat_store_dword v[0:1], v2
2847 ; GFX7-NEXT: s_endpgm
2848 ;
2849 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2850 ; GFX10-WGP: ; %bb.0: ; %entry
2851 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
2852 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2853 ; GFX10-WGP-NEXT: s_nop 0
2854 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
2855 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
2856 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
2857 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2858 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2859 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2860 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
2861 ; GFX10-WGP-NEXT: buffer_gl0_inv
2862 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
2863 ; GFX10-WGP-NEXT: s_endpgm
2864 ;
2865 ; GFX10-CU-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2866 ; GFX10-CU: ; %bb.0: ; %entry
2867 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
2868 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2869 ; GFX10-CU-NEXT: s_nop 0
2870 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
2871 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2872 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
2873 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
2874 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2875 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
2876 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
2877 ; GFX10-CU-NEXT: s_endpgm
2878 ;
2879 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2880 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
2881 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2882 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
2883 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2884 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
2885 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
2886 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
2887 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
2888 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
2889 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
2890 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
2891 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
2892 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
2893 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
2894 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
2895 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
2896 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
2897 ; SKIP-CACHE-INV-NEXT: s_endpgm
2898 ;
2899 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2900 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
2901 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2902 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2903 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
2904 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2905 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2906 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2907 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2908 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2909 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2910 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2911 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
2912 ;
2913 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2914 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
2915 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2916 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
2917 ; GFX90A-TGSPLIT-NEXT: s_nop 0
2918 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
2919 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2920 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
2921 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2922 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
2923 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2924 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
2925 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
2926 ; GFX90A-TGSPLIT-NEXT: s_endpgm
2927 ;
2928 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2929 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
2930 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2931 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2932 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
2933 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2934 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2935 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2936 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2937 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2938 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
2939 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2940 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
2941 ;
2942 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2943 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
2944 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
2945 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
2946 ; GFX940-TGSPLIT-NEXT: s_nop 0
2947 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
2948 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
2949 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
2950 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2951 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
2952 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
2953 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
2954 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
2955 ; GFX940-TGSPLIT-NEXT: s_endpgm
2956 ;
2957 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2958 ; GFX11-WGP: ; %bb.0: ; %entry
2959 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
2960 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2961 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2962 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
2963 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
2964 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
2965 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
2966 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2967 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
2968 ; GFX11-WGP-NEXT: buffer_gl0_inv
2969 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
2970 ; GFX11-WGP-NEXT: s_endpgm
2971 ;
2972 ; GFX11-CU-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2973 ; GFX11-CU: ; %bb.0: ; %entry
2974 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
2975 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2976 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
2977 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2978 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
2979 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
2980 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
2981 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
2982 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
2983 ; GFX11-CU-NEXT: s_endpgm
2984 ;
2985 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
2986 ; GFX12-WGP: ; %bb.0: ; %entry
2987 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
2988 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
2989 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
2990 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
2991 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
2992 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
2993 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2994 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
2995 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
2996 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
2997 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
2998 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
2999 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3000 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
3001 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
3002 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
3003 ; GFX12-WGP-NEXT: s_endpgm
3004 ;
3005 ; GFX12-CU-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
3006 ; GFX12-CU: ; %bb.0: ; %entry
3007 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
3008 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
3009 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
3010 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3011 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
3012 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3013 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
3014 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
3015 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
3016 ; GFX12-CU-NEXT: s_endpgm
3017 ptr addrspace(1) %out, i32 %in) {
3018 entry:
3019 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
3020 store i32 %val, ptr addrspace(1) %out, align 4
3021 ret void
3022 }
3024 define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
3025 ; GFX6-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3026 ; GFX6: ; %bb.0: ; %entry
3027 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
3028 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
3029 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
3030 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3031 ; GFX6-NEXT: s_mov_b32 s12, s5
3032 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
3033 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
3034 ; GFX6-NEXT: s_mov_b32 s11, -1
3035 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
3036 ; GFX6-NEXT: s_mov_b32 s5, s12
3037 ; GFX6-NEXT: s_mov_b32 s6, s11
3038 ; GFX6-NEXT: s_mov_b32 s7, s10
3039 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
3040 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
3041 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3042 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3043 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
3044 ; GFX6-NEXT: s_endpgm
3045 ;
3046 ; GFX7-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3047 ; GFX7: ; %bb.0: ; %entry
3048 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3049 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3050 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3051 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3052 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3053 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3054 ; GFX7-NEXT: s_mov_b32 s4, s8
3055 ; GFX7-NEXT: s_mov_b32 s5, s9
3056 ; GFX7-NEXT: s_mov_b32 s9, s10
3057 ; GFX7-NEXT: s_mov_b32 s8, s11
3058 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3059 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3060 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3061 ; GFX7-NEXT: s_mov_b32 s5, s8
3062 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3063 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3064 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3065 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3066 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3067 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3068 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3069 ; GFX7-NEXT: s_endpgm
3070 ;
3071 ; GFX10-WGP-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3072 ; GFX10-WGP: ; %bb.0: ; %entry
3073 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
3074 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
3075 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3076 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
3077 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
3078 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3079 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
3080 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
3081 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3082 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
3083 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3084 ; GFX10-WGP-NEXT: s_endpgm
3085 ;
3086 ; GFX10-CU-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3087 ; GFX10-CU: ; %bb.0: ; %entry
3088 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
3089 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
3090 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3091 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
3092 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
3093 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3094 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
3095 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
3096 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3097 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
3098 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3099 ; GFX10-CU-NEXT: s_endpgm
3100 ;
3101 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3102 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3103 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
3104 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
3105 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
3106 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3107 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
3108 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
3109 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
3110 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
3111 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
3112 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
3113 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
3114 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
3115 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
3116 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
3117 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3118 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
3119 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3120 ; SKIP-CACHE-INV-NEXT: s_endpgm
3121 ;
3122 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3123 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3124 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3125 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3126 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3127 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3128 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3129 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3130 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3131 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3132 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3133 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3134 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3135 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3136 ;
3137 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3138 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3139 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3140 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3141 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3142 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3143 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3144 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3145 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3146 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3147 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3148 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3149 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3150 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3151 ;
3152 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3153 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3154 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3155 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3156 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3157 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3158 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3159 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3160 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3161 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3162 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3164 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3165 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3166 ;
3167 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3168 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3169 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3170 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3171 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3172 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3173 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3174 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3175 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3176 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3177 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3178 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3179 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3180 ; GFX940-TGSPLIT-NEXT: s_endpgm
3181 ;
3182 ; GFX11-WGP-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3183 ; GFX11-WGP: ; %bb.0: ; %entry
3184 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3185 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
3186 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3187 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3188 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3189 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3190 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
3191 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
3192 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3193 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
3194 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3195 ; GFX11-WGP-NEXT: s_endpgm
3196 ;
3197 ; GFX11-CU-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3198 ; GFX11-CU: ; %bb.0: ; %entry
3199 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3200 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
3201 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3202 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3203 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3204 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3205 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
3206 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
3207 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3208 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
3209 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3210 ; GFX11-CU-NEXT: s_endpgm
3211 ;
3212 ; GFX12-WGP-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3213 ; GFX12-WGP: ; %bb.0: ; %entry
3214 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3215 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
3216 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3217 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3218 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3219 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3220 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
3221 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
3222 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3223 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
3224 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
3225 ; GFX12-WGP-NEXT: s_endpgm
3226 ;
3227 ; GFX12-CU-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
3228 ; GFX12-CU: ; %bb.0: ; %entry
3229 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3230 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
3231 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3232 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3233 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3234 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3235 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
3236 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
3237 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3238 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
3239 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3240 ; GFX12-CU-NEXT: s_endpgm
3241 ptr addrspace(1) %out, i32 %in, i32 %old) {
3242 entry:
3243 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
3244 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
3245 ret void
3246 }
3248 define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
3249 ; GFX6-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3250 ; GFX6: ; %bb.0: ; %entry
3251 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
3252 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
3253 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
3254 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3255 ; GFX6-NEXT: s_mov_b32 s12, s5
3256 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
3257 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
3258 ; GFX6-NEXT: s_mov_b32 s11, -1
3259 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
3260 ; GFX6-NEXT: s_mov_b32 s5, s12
3261 ; GFX6-NEXT: s_mov_b32 s6, s11
3262 ; GFX6-NEXT: s_mov_b32 s7, s10
3263 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
3264 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
3265 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3266 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3267 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
3268 ; GFX6-NEXT: s_endpgm
3269 ;
3270 ; GFX7-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3271 ; GFX7: ; %bb.0: ; %entry
3272 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3273 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3274 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3275 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3276 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3277 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3278 ; GFX7-NEXT: s_mov_b32 s4, s8
3279 ; GFX7-NEXT: s_mov_b32 s5, s9
3280 ; GFX7-NEXT: s_mov_b32 s9, s10
3281 ; GFX7-NEXT: s_mov_b32 s8, s11
3282 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3283 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3284 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3285 ; GFX7-NEXT: s_mov_b32 s5, s8
3286 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3287 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3288 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3289 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3290 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3291 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3292 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3293 ; GFX7-NEXT: s_endpgm
3294 ;
3295 ; GFX10-WGP-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3296 ; GFX10-WGP: ; %bb.0: ; %entry
3297 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
3298 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
3299 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3300 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
3301 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
3302 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3303 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
3304 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
3305 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3306 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
3307 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3308 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3309 ; GFX10-WGP-NEXT: buffer_gl0_inv
3310 ; GFX10-WGP-NEXT: s_endpgm
3311 ;
3312 ; GFX10-CU-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3313 ; GFX10-CU: ; %bb.0: ; %entry
3314 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
3315 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
3316 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3317 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
3318 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
3319 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3320 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
3321 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
3322 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3323 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
3324 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3325 ; GFX10-CU-NEXT: s_endpgm
3326 ;
3327 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3328 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3329 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
3330 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
3331 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
3332 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3333 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
3334 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
3335 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
3336 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
3337 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
3338 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
3339 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
3340 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
3341 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
3342 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
3343 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3344 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
3345 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3346 ; SKIP-CACHE-INV-NEXT: s_endpgm
3347 ;
3348 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3349 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3350 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3351 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3352 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3353 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3354 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3355 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3356 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3357 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3358 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3359 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3360 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3361 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3362 ;
3363 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3364 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3365 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3366 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3367 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3368 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3369 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3370 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3371 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3372 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3373 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3374 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3375 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3376 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3377 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3378 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3379 ;
3380 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3381 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3382 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3383 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3384 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3385 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3386 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3387 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3388 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3389 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3390 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3391 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3392 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3393 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3394 ;
3395 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3396 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3397 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3398 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3399 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3400 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3401 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3402 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3403 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3404 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3405 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3406 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3407 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3408 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3409 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
3410 ; GFX940-TGSPLIT-NEXT: s_endpgm
3411 ;
3412 ; GFX11-WGP-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3413 ; GFX11-WGP: ; %bb.0: ; %entry
3414 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3415 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
3416 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3417 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3418 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3419 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3420 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
3421 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
3422 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3423 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
3424 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3425 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3426 ; GFX11-WGP-NEXT: buffer_gl0_inv
3427 ; GFX11-WGP-NEXT: s_endpgm
3428 ;
3429 ; GFX11-CU-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3430 ; GFX11-CU: ; %bb.0: ; %entry
3431 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3432 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
3433 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3434 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3435 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3436 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3437 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
3438 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
3439 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3440 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
3441 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3442 ; GFX11-CU-NEXT: s_endpgm
3443 ;
3444 ; GFX12-WGP-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3445 ; GFX12-WGP: ; %bb.0: ; %entry
3446 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3447 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
3448 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3449 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3450 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3451 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3452 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
3453 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
3454 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3455 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
3456 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
3457 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3458 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
3459 ; GFX12-WGP-NEXT: s_endpgm
3460 ;
3461 ; GFX12-CU-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
3462 ; GFX12-CU: ; %bb.0: ; %entry
3463 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3464 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
3465 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3466 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3467 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3468 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3469 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
3470 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
3471 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3472 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
3473 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3474 ; GFX12-CU-NEXT: s_endpgm
3475 ptr addrspace(1) %out, i32 %in, i32 %old) {
3476 entry:
3477 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
3478 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
3479 ret void
3480 }
3482 define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
3483 ; GFX6-LABEL: global_workgroup_release_monotonic_cmpxchg:
3484 ; GFX6: ; %bb.0: ; %entry
3485 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
3486 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
3487 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
3488 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3489 ; GFX6-NEXT: s_mov_b32 s12, s5
3490 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
3491 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
3492 ; GFX6-NEXT: s_mov_b32 s11, -1
3493 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
3494 ; GFX6-NEXT: s_mov_b32 s5, s12
3495 ; GFX6-NEXT: s_mov_b32 s6, s11
3496 ; GFX6-NEXT: s_mov_b32 s7, s10
3497 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
3498 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
3499 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3500 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3501 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3502 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
3503 ; GFX6-NEXT: s_endpgm
3504 ;
3505 ; GFX7-LABEL: global_workgroup_release_monotonic_cmpxchg:
3506 ; GFX7: ; %bb.0: ; %entry
3507 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3508 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3509 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3510 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3511 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3512 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3513 ; GFX7-NEXT: s_mov_b32 s4, s8
3514 ; GFX7-NEXT: s_mov_b32 s5, s9
3515 ; GFX7-NEXT: s_mov_b32 s9, s10
3516 ; GFX7-NEXT: s_mov_b32 s8, s11
3517 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3518 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3519 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3520 ; GFX7-NEXT: s_mov_b32 s5, s8
3521 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3522 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3523 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3524 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3525 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3526 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3527 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3528 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3529 ; GFX7-NEXT: s_endpgm
3530 ;
3531 ; GFX10-WGP-LABEL: global_workgroup_release_monotonic_cmpxchg:
3532 ; GFX10-WGP: ; %bb.0: ; %entry
3533 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
3534 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
3535 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3536 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
3537 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
3538 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3539 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
3540 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
3541 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3542 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
3543 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3544 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3545 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3546 ; GFX10-WGP-NEXT: s_endpgm
3547 ;
3548 ; GFX10-CU-LABEL: global_workgroup_release_monotonic_cmpxchg:
3549 ; GFX10-CU: ; %bb.0: ; %entry
3550 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
3551 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
3552 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3553 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
3554 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
3555 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3556 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
3557 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
3558 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3559 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
3560 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3561 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3562 ; GFX10-CU-NEXT: s_endpgm
3563 ;
3564 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_monotonic_cmpxchg:
3565 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3566 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
3567 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
3568 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
3569 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3570 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
3571 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
3572 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
3573 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
3574 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
3575 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
3576 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
3577 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
3578 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
3579 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
3580 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3581 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
3582 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3583 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3584 ; SKIP-CACHE-INV-NEXT: s_endpgm
3585 ;
3586 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_monotonic_cmpxchg:
3587 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3588 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3589 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3590 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3591 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3592 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3593 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3594 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3595 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3596 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3597 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3598 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3599 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3600 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3601 ;
3602 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_monotonic_cmpxchg:
3603 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3604 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3605 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3606 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3607 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3608 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3609 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3610 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3611 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3612 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3613 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3614 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3615 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3616 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3617 ;
3618 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_monotonic_cmpxchg:
3619 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3620 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3621 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3622 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3623 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3624 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3625 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3626 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3627 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3628 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3629 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3630 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3631 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3632 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3633 ;
3634 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_monotonic_cmpxchg:
3635 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3636 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3637 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3638 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3639 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3640 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3641 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3642 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3643 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3644 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3645 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3646 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3647 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3648 ; GFX940-TGSPLIT-NEXT: s_endpgm
3649 ;
3650 ; GFX11-WGP-LABEL: global_workgroup_release_monotonic_cmpxchg:
3651 ; GFX11-WGP: ; %bb.0: ; %entry
3652 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3653 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
3654 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3655 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3656 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3657 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3658 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
3659 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
3660 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3661 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
3662 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3663 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3664 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3665 ; GFX11-WGP-NEXT: s_endpgm
3666 ;
3667 ; GFX11-CU-LABEL: global_workgroup_release_monotonic_cmpxchg:
3668 ; GFX11-CU: ; %bb.0: ; %entry
3669 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3670 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
3671 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3672 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3673 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3674 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3675 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
3676 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
3677 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3678 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
3679 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3680 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3681 ; GFX11-CU-NEXT: s_endpgm
3682 ;
3683 ; GFX12-WGP-LABEL: global_workgroup_release_monotonic_cmpxchg:
3684 ; GFX12-WGP: ; %bb.0: ; %entry
3685 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3686 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
3687 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3688 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3689 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3690 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3691 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
3692 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
3693 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3694 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
3695 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
3696 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3697 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3698 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3699 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3700 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
3701 ; GFX12-WGP-NEXT: s_endpgm
3702 ;
3703 ; GFX12-CU-LABEL: global_workgroup_release_monotonic_cmpxchg:
3704 ; GFX12-CU: ; %bb.0: ; %entry
3705 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3706 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
3707 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3708 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3709 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3710 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3711 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
3712 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
3713 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3714 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
3715 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3716 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3717 ; GFX12-CU-NEXT: s_endpgm
3718 ptr addrspace(1) %out, i32 %in, i32 %old) {
3719 entry:
3720 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
3721 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
3722 ret void
3723 }
3725 define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
3726 ; GFX6-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3727 ; GFX6: ; %bb.0: ; %entry
3728 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
3729 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
3730 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
3731 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3732 ; GFX6-NEXT: s_mov_b32 s12, s5
3733 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
3734 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
3735 ; GFX6-NEXT: s_mov_b32 s11, -1
3736 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
3737 ; GFX6-NEXT: s_mov_b32 s5, s12
3738 ; GFX6-NEXT: s_mov_b32 s6, s11
3739 ; GFX6-NEXT: s_mov_b32 s7, s10
3740 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
3741 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
3742 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3743 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3744 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3745 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
3746 ; GFX6-NEXT: s_endpgm
3747 ;
3748 ; GFX7-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3749 ; GFX7: ; %bb.0: ; %entry
3750 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
3751 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
3752 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
3753 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
3754 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
3755 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3756 ; GFX7-NEXT: s_mov_b32 s4, s8
3757 ; GFX7-NEXT: s_mov_b32 s5, s9
3758 ; GFX7-NEXT: s_mov_b32 s9, s10
3759 ; GFX7-NEXT: s_mov_b32 s8, s11
3760 ; GFX7-NEXT: s_add_u32 s4, s4, s9
3761 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
3762 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
3763 ; GFX7-NEXT: s_mov_b32 s5, s8
3764 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
3765 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
3766 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3767 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
3768 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
3769 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
3770 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3771 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
3772 ; GFX7-NEXT: s_endpgm
3773 ;
3774 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3775 ; GFX10-WGP: ; %bb.0: ; %entry
3776 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
3777 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
3778 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3779 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
3780 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
3781 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
3782 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
3783 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
3784 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3785 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
3786 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3787 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3788 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3789 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3790 ; GFX10-WGP-NEXT: buffer_gl0_inv
3791 ; GFX10-WGP-NEXT: s_endpgm
3792 ;
3793 ; GFX10-CU-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3794 ; GFX10-CU: ; %bb.0: ; %entry
3795 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
3796 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
3797 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3798 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
3799 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
3800 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3801 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
3802 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
3803 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3804 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
3805 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
3806 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
3807 ; GFX10-CU-NEXT: s_endpgm
3808 ;
3809 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3810 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
3811 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
3812 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
3813 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
3814 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3815 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
3816 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
3817 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
3818 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
3819 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
3820 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
3821 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
3822 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
3823 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
3824 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
3825 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3826 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
3827 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
3828 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
3829 ; SKIP-CACHE-INV-NEXT: s_endpgm
3830 ;
3831 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3832 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
3833 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3834 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3835 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3836 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3837 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3838 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3839 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3840 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3841 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3842 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3843 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3844 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3845 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
3846 ;
3847 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3848 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
3849 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
3850 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3851 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
3852 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
3853 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
3854 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3855 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
3856 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
3857 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3858 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3859 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3860 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
3861 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3862 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
3863 ; GFX90A-TGSPLIT-NEXT: s_endpgm
3864 ;
3865 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3866 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
3867 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3868 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3869 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3870 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3871 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3872 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3873 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3874 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3875 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3876 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3877 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3878 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3879 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
3880 ;
3881 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3882 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
3883 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
3884 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
3885 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
3886 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
3887 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
3888 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
3889 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
3890 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
3891 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
3892 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
3893 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3894 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
3895 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
3896 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
3897 ; GFX940-TGSPLIT-NEXT: s_endpgm
3898 ;
3899 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3900 ; GFX11-WGP: ; %bb.0: ; %entry
3901 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3902 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
3903 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3904 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3905 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3906 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
3907 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
3908 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
3909 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3910 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
3911 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
3912 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3913 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3914 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
3915 ; GFX11-WGP-NEXT: buffer_gl0_inv
3916 ; GFX11-WGP-NEXT: s_endpgm
3917 ;
3918 ; GFX11-CU-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3919 ; GFX11-CU: ; %bb.0: ; %entry
3920 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3921 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
3922 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3923 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3924 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3925 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3926 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
3927 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
3928 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3929 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
3930 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
3931 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3932 ; GFX11-CU-NEXT: s_endpgm
3933 ;
3934 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3935 ; GFX12-WGP: ; %bb.0: ; %entry
3936 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
3937 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
3938 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3939 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
3940 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
3941 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
3942 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
3943 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
3944 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3945 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
3946 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
3947 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
3948 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
3949 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3950 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
3951 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
3952 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
3953 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
3954 ; GFX12-WGP-NEXT: s_endpgm
3955 ;
3956 ; GFX12-CU-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
3957 ; GFX12-CU: ; %bb.0: ; %entry
3958 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
3959 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
3960 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
3961 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
3962 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
3963 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
3964 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
3965 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
3966 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
3967 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
3968 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
3969 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
3970 ; GFX12-CU-NEXT: s_endpgm
3971 ptr addrspace(1) %out, i32 %in, i32 %old) {
3972 entry:
3973 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
3974 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
3975 ret void
3976 }
3978 define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
3979 ; GFX6-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
3980 ; GFX6: ; %bb.0: ; %entry
3981 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
3982 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
3983 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
3984 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3985 ; GFX6-NEXT: s_mov_b32 s12, s5
3986 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
3987 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
3988 ; GFX6-NEXT: s_mov_b32 s11, -1
3989 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
3990 ; GFX6-NEXT: s_mov_b32 s5, s12
3991 ; GFX6-NEXT: s_mov_b32 s6, s11
3992 ; GFX6-NEXT: s_mov_b32 s7, s10
3993 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
3994 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
3995 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
3996 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
3997 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
3998 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
3999 ; GFX6-NEXT: s_endpgm
4000 ;
4001 ; GFX7-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4002 ; GFX7: ; %bb.0: ; %entry
4003 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4004 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4005 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4006 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4007 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4008 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4009 ; GFX7-NEXT: s_mov_b32 s4, s8
4010 ; GFX7-NEXT: s_mov_b32 s5, s9
4011 ; GFX7-NEXT: s_mov_b32 s9, s10
4012 ; GFX7-NEXT: s_mov_b32 s8, s11
4013 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4014 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4015 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4016 ; GFX7-NEXT: s_mov_b32 s5, s8
4017 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4018 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4019 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4020 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4021 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4022 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4023 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4024 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4025 ; GFX7-NEXT: s_endpgm
4026 ;
4027 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4028 ; GFX10-WGP: ; %bb.0: ; %entry
4029 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
4030 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
4031 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4032 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
4033 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
4034 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4035 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
4036 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
4037 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4038 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
4039 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4040 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4041 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4042 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4043 ; GFX10-WGP-NEXT: buffer_gl0_inv
4044 ; GFX10-WGP-NEXT: s_endpgm
4045 ;
4046 ; GFX10-CU-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4047 ; GFX10-CU: ; %bb.0: ; %entry
4048 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
4049 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
4050 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4051 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
4052 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
4053 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4054 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
4055 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
4056 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4057 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
4058 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4059 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4060 ; GFX10-CU-NEXT: s_endpgm
4061 ;
4062 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4063 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4064 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
4065 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
4066 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
4067 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4068 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
4069 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
4070 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
4071 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
4072 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
4073 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
4074 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
4075 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
4076 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
4077 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
4078 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4079 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
4080 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4081 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4082 ; SKIP-CACHE-INV-NEXT: s_endpgm
4083 ;
4084 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4085 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4086 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4087 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4088 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4089 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4090 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4091 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4092 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4093 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4094 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4095 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4096 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4097 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4098 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4099 ;
4100 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4101 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4102 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4103 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4104 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4105 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4106 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4107 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4108 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4109 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4110 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4111 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4112 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4113 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4114 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4115 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4116 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4117 ;
4118 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4119 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4120 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4121 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4122 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4123 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4124 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4125 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4126 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4127 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4128 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4129 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4130 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4131 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4132 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4133 ;
4134 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4135 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4136 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4137 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4138 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4139 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4140 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4141 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4142 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4143 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4144 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4145 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4146 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4147 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4148 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4149 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4150 ; GFX940-TGSPLIT-NEXT: s_endpgm
4151 ;
4152 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4153 ; GFX11-WGP: ; %bb.0: ; %entry
4154 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4155 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
4156 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4157 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4158 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4159 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4160 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
4161 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
4162 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4163 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
4164 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4165 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4166 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4167 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4168 ; GFX11-WGP-NEXT: buffer_gl0_inv
4169 ; GFX11-WGP-NEXT: s_endpgm
4170 ;
4171 ; GFX11-CU-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4172 ; GFX11-CU: ; %bb.0: ; %entry
4173 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4174 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
4175 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4176 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4177 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4178 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4179 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
4180 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
4181 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4182 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
4183 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4184 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4185 ; GFX11-CU-NEXT: s_endpgm
4186 ;
4187 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4188 ; GFX12-WGP: ; %bb.0: ; %entry
4189 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4190 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
4191 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4192 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4193 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4194 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4195 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
4196 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
4197 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4198 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
4199 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
4200 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4201 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4202 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4203 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4204 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
4205 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4206 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4207 ; GFX12-WGP-NEXT: s_endpgm
4208 ;
4209 ; GFX12-CU-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
4210 ; GFX12-CU: ; %bb.0: ; %entry
4211 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4212 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
4213 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4214 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4215 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4216 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4217 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
4218 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
4219 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4220 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
4221 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4222 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4223 ; GFX12-CU-NEXT: s_endpgm
4224 ptr addrspace(1) %out, i32 %in, i32 %old) {
4225 entry:
4226 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
4227 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
4228 ret void
4229 }
4231 define amdgpu_kernel void @global_workgroup_monotonic_acquire_cmpxchg(
4232 ; GFX6-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4233 ; GFX6: ; %bb.0: ; %entry
4234 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
4235 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
4236 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
4237 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4238 ; GFX6-NEXT: s_mov_b32 s12, s5
4239 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
4240 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
4241 ; GFX6-NEXT: s_mov_b32 s11, -1
4242 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
4243 ; GFX6-NEXT: s_mov_b32 s5, s12
4244 ; GFX6-NEXT: s_mov_b32 s6, s11
4245 ; GFX6-NEXT: s_mov_b32 s7, s10
4246 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
4247 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
4248 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4249 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
4250 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4251 ; GFX6-NEXT: s_endpgm
4252 ;
4253 ; GFX7-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4254 ; GFX7: ; %bb.0: ; %entry
4255 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4256 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4257 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4258 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4259 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4260 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4261 ; GFX7-NEXT: s_mov_b32 s4, s8
4262 ; GFX7-NEXT: s_mov_b32 s5, s9
4263 ; GFX7-NEXT: s_mov_b32 s9, s10
4264 ; GFX7-NEXT: s_mov_b32 s8, s11
4265 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4266 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4267 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4268 ; GFX7-NEXT: s_mov_b32 s5, s8
4269 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4270 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4271 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4272 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4273 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4274 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4275 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4276 ; GFX7-NEXT: s_endpgm
4277 ;
4278 ; GFX10-WGP-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4279 ; GFX10-WGP: ; %bb.0: ; %entry
4280 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
4281 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
4282 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4283 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
4284 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
4285 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4286 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
4287 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
4288 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4289 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
4290 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4291 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4292 ; GFX10-WGP-NEXT: buffer_gl0_inv
4293 ; GFX10-WGP-NEXT: s_endpgm
4294 ;
4295 ; GFX10-CU-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4296 ; GFX10-CU: ; %bb.0: ; %entry
4297 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
4298 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
4299 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4300 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
4301 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
4302 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4303 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
4304 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
4305 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4306 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
4307 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4308 ; GFX10-CU-NEXT: s_endpgm
4309 ;
4310 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4311 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4312 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
4313 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
4314 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
4315 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4316 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
4317 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
4318 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
4319 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
4320 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
4321 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
4322 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
4323 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
4324 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
4325 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
4326 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4327 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
4328 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4329 ; SKIP-CACHE-INV-NEXT: s_endpgm
4330 ;
4331 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4332 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4333 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4334 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4335 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4336 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4337 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4338 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4339 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4340 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4341 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4342 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4343 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4344 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4345 ;
4346 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4347 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4348 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4349 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4350 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4351 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4352 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4353 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4354 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4355 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4356 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4357 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4358 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4359 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4360 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4361 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4362 ;
4363 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4364 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4365 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4366 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4367 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4368 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4369 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4370 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4371 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4372 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4373 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4374 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4375 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4376 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4377 ;
4378 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4379 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4380 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4381 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4382 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4383 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4384 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4385 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4386 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4387 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4388 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4389 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4390 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4391 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4392 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4393 ; GFX940-TGSPLIT-NEXT: s_endpgm
4394 ;
4395 ; GFX11-WGP-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4396 ; GFX11-WGP: ; %bb.0: ; %entry
4397 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4398 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
4399 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4400 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4401 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4402 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4403 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
4404 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
4405 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4406 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
4407 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4408 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4409 ; GFX11-WGP-NEXT: buffer_gl0_inv
4410 ; GFX11-WGP-NEXT: s_endpgm
4411 ;
4412 ; GFX11-CU-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4413 ; GFX11-CU: ; %bb.0: ; %entry
4414 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4415 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
4416 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4417 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4418 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4419 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4420 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
4421 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
4422 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4423 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
4424 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4425 ; GFX11-CU-NEXT: s_endpgm
4426 ;
4427 ; GFX12-WGP-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4428 ; GFX12-WGP: ; %bb.0: ; %entry
4429 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4430 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
4431 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4432 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4433 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4434 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
4436 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
4437 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4438 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
4439 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
4440 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4441 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4442 ; GFX12-WGP-NEXT: s_endpgm
4443 ;
4444 ; GFX12-CU-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
4445 ; GFX12-CU: ; %bb.0: ; %entry
4446 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4447 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
4448 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4449 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4450 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4451 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4452 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
4453 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
4454 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4455 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
4456 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4457 ; GFX12-CU-NEXT: s_endpgm
4458 ptr addrspace(1) %out, i32 %in, i32 %old) {
4459 entry:
4460 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
4461 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
4462 ret void
4463 }
4465 define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
4466 ; GFX6-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4467 ; GFX6: ; %bb.0: ; %entry
4468 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
4469 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
4470 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
4471 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4472 ; GFX6-NEXT: s_mov_b32 s12, s5
4473 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
4474 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
4475 ; GFX6-NEXT: s_mov_b32 s11, -1
4476 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
4477 ; GFX6-NEXT: s_mov_b32 s5, s12
4478 ; GFX6-NEXT: s_mov_b32 s6, s11
4479 ; GFX6-NEXT: s_mov_b32 s7, s10
4480 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
4481 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
4482 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4483 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
4484 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4485 ; GFX6-NEXT: s_endpgm
4486 ;
4487 ; GFX7-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4488 ; GFX7: ; %bb.0: ; %entry
4489 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4490 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4491 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4492 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4493 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4494 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4495 ; GFX7-NEXT: s_mov_b32 s4, s8
4496 ; GFX7-NEXT: s_mov_b32 s5, s9
4497 ; GFX7-NEXT: s_mov_b32 s9, s10
4498 ; GFX7-NEXT: s_mov_b32 s8, s11
4499 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4500 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4501 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4502 ; GFX7-NEXT: s_mov_b32 s5, s8
4503 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4504 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4505 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4506 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4507 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4508 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4509 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4510 ; GFX7-NEXT: s_endpgm
4511 ;
4512 ; GFX10-WGP-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4513 ; GFX10-WGP: ; %bb.0: ; %entry
4514 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
4515 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
4516 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4517 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
4518 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
4519 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4520 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
4521 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
4522 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4523 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
4524 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4525 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4526 ; GFX10-WGP-NEXT: buffer_gl0_inv
4527 ; GFX10-WGP-NEXT: s_endpgm
4528 ;
4529 ; GFX10-CU-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4530 ; GFX10-CU: ; %bb.0: ; %entry
4531 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
4532 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
4533 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4534 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
4535 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
4536 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4537 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
4538 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
4539 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4540 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
4541 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4542 ; GFX10-CU-NEXT: s_endpgm
4543 ;
4544 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4545 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4546 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
4547 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
4548 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
4549 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4550 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
4551 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
4552 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
4553 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
4554 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
4555 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
4556 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
4557 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
4558 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
4559 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
4560 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4561 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
4562 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4563 ; SKIP-CACHE-INV-NEXT: s_endpgm
4564 ;
4565 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4566 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4567 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4568 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4569 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4570 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4571 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4572 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4573 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4574 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4575 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4576 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4577 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4578 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4579 ;
4580 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4581 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4582 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4583 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4584 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4585 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4586 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4587 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4588 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4589 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4590 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4591 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4592 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4593 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4594 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4595 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4596 ;
4597 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4598 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4599 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4600 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4601 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4602 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4603 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4604 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4605 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4606 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4607 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4608 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4609 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4610 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4611 ;
4612 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4613 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4614 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4615 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4616 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4617 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4618 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4619 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4620 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4621 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4622 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4623 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4624 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4625 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4626 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4627 ; GFX940-TGSPLIT-NEXT: s_endpgm
4628 ;
4629 ; GFX11-WGP-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4630 ; GFX11-WGP: ; %bb.0: ; %entry
4631 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4632 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
4633 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4634 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4635 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4636 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4637 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
4638 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
4639 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4640 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
4641 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4642 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4643 ; GFX11-WGP-NEXT: buffer_gl0_inv
4644 ; GFX11-WGP-NEXT: s_endpgm
4645 ;
4646 ; GFX11-CU-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4647 ; GFX11-CU: ; %bb.0: ; %entry
4648 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4649 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
4650 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4651 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4652 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4653 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4654 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
4655 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
4656 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4657 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
4658 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4659 ; GFX11-CU-NEXT: s_endpgm
4660 ;
4661 ; GFX12-WGP-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4662 ; GFX12-WGP: ; %bb.0: ; %entry
4663 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4664 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
4665 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4666 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4667 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4668 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4669 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
4670 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
4671 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4672 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
4673 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
4674 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4675 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4676 ; GFX12-WGP-NEXT: s_endpgm
4677 ;
4678 ; GFX12-CU-LABEL: global_workgroup_acquire_acquire_cmpxchg:
4679 ; GFX12-CU: ; %bb.0: ; %entry
4680 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4681 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
4682 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4683 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4684 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4685 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4686 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
4687 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
4688 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4689 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
4690 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4691 ; GFX12-CU-NEXT: s_endpgm
4692 ptr addrspace(1) %out, i32 %in, i32 %old) {
4693 entry:
4694 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
4695 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
4696 ret void
4697 }
4699 define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
4700 ; GFX6-LABEL: global_workgroup_release_acquire_cmpxchg:
4701 ; GFX6: ; %bb.0: ; %entry
4702 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
4703 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
4704 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
4705 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4706 ; GFX6-NEXT: s_mov_b32 s12, s5
4707 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
4708 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
4709 ; GFX6-NEXT: s_mov_b32 s11, -1
4710 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
4711 ; GFX6-NEXT: s_mov_b32 s5, s12
4712 ; GFX6-NEXT: s_mov_b32 s6, s11
4713 ; GFX6-NEXT: s_mov_b32 s7, s10
4714 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
4715 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
4716 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4717 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
4718 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4719 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4720 ; GFX6-NEXT: s_endpgm
4721 ;
4722 ; GFX7-LABEL: global_workgroup_release_acquire_cmpxchg:
4723 ; GFX7: ; %bb.0: ; %entry
4724 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4725 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4726 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4727 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4728 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4729 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4730 ; GFX7-NEXT: s_mov_b32 s4, s8
4731 ; GFX7-NEXT: s_mov_b32 s5, s9
4732 ; GFX7-NEXT: s_mov_b32 s9, s10
4733 ; GFX7-NEXT: s_mov_b32 s8, s11
4734 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4735 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4736 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4737 ; GFX7-NEXT: s_mov_b32 s5, s8
4738 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4739 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4740 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4741 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4742 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4743 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4744 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4745 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4746 ; GFX7-NEXT: s_endpgm
4747 ;
4748 ; GFX10-WGP-LABEL: global_workgroup_release_acquire_cmpxchg:
4749 ; GFX10-WGP: ; %bb.0: ; %entry
4750 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
4751 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
4752 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4753 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
4754 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
4755 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
4756 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
4757 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
4758 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4759 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
4760 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4761 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4762 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4763 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4764 ; GFX10-WGP-NEXT: buffer_gl0_inv
4765 ; GFX10-WGP-NEXT: s_endpgm
4766 ;
4767 ; GFX10-CU-LABEL: global_workgroup_release_acquire_cmpxchg:
4768 ; GFX10-CU: ; %bb.0: ; %entry
4769 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
4770 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
4771 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4772 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
4773 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
4774 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4775 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
4776 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
4777 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4778 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
4779 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
4780 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
4781 ; GFX10-CU-NEXT: s_endpgm
4782 ;
4783 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_acquire_cmpxchg:
4784 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
4785 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
4786 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
4787 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
4788 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4789 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
4790 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
4791 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
4792 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
4793 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
4794 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
4795 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
4796 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
4797 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
4798 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
4799 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4800 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
4801 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
4802 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
4803 ; SKIP-CACHE-INV-NEXT: s_endpgm
4804 ;
4805 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_acquire_cmpxchg:
4806 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
4807 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4808 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4809 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4810 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4811 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4812 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4813 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4814 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4815 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4816 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4817 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4818 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4819 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
4820 ;
4821 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_acquire_cmpxchg:
4822 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
4823 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
4824 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4825 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
4826 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
4827 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
4828 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4829 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
4830 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
4831 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4832 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4833 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4834 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
4835 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4836 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
4837 ; GFX90A-TGSPLIT-NEXT: s_endpgm
4838 ;
4839 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_acquire_cmpxchg:
4840 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
4841 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4842 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4843 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4844 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4845 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4846 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4847 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4848 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4849 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4850 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4851 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4852 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4853 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
4854 ;
4855 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_acquire_cmpxchg:
4856 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
4857 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
4858 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
4859 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
4860 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
4861 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
4862 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
4863 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
4864 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
4865 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4866 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
4867 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4868 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
4869 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
4870 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
4871 ; GFX940-TGSPLIT-NEXT: s_endpgm
4872 ;
4873 ; GFX11-WGP-LABEL: global_workgroup_release_acquire_cmpxchg:
4874 ; GFX11-WGP: ; %bb.0: ; %entry
4875 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4876 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
4877 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4878 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4879 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4880 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
4881 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
4882 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
4883 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4884 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
4885 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
4886 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4887 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4888 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
4889 ; GFX11-WGP-NEXT: buffer_gl0_inv
4890 ; GFX11-WGP-NEXT: s_endpgm
4891 ;
4892 ; GFX11-CU-LABEL: global_workgroup_release_acquire_cmpxchg:
4893 ; GFX11-CU: ; %bb.0: ; %entry
4894 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4895 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
4896 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4897 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4898 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4899 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4900 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
4901 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
4902 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4903 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
4904 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
4905 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4906 ; GFX11-CU-NEXT: s_endpgm
4907 ;
4908 ; GFX12-WGP-LABEL: global_workgroup_release_acquire_cmpxchg:
4909 ; GFX12-WGP: ; %bb.0: ; %entry
4910 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
4911 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
4912 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4913 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
4914 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
4915 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
4916 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
4917 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
4918 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4919 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
4920 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
4921 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
4922 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
4923 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4924 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
4925 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
4926 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
4927 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
4928 ; GFX12-WGP-NEXT: s_endpgm
4929 ;
4930 ; GFX12-CU-LABEL: global_workgroup_release_acquire_cmpxchg:
4931 ; GFX12-CU: ; %bb.0: ; %entry
4932 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
4933 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
4934 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
4935 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
4936 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
4937 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
4938 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
4939 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
4940 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
4941 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
4942 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
4943 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
4944 ; GFX12-CU-NEXT: s_endpgm
4945 ptr addrspace(1) %out, i32 %in, i32 %old) {
4946 entry:
4947 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
4948 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
4949 ret void
4950 }
4952 define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
4953 ; GFX6-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
4954 ; GFX6: ; %bb.0: ; %entry
4955 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
4956 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
4957 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
4958 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4959 ; GFX6-NEXT: s_mov_b32 s12, s5
4960 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
4961 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
4962 ; GFX6-NEXT: s_mov_b32 s11, -1
4963 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
4964 ; GFX6-NEXT: s_mov_b32 s5, s12
4965 ; GFX6-NEXT: s_mov_b32 s6, s11
4966 ; GFX6-NEXT: s_mov_b32 s7, s10
4967 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
4968 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
4969 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
4970 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
4971 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
4972 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
4973 ; GFX6-NEXT: s_endpgm
4974 ;
4975 ; GFX7-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
4976 ; GFX7: ; %bb.0: ; %entry
4977 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4978 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
4979 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
4980 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
4981 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
4982 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4983 ; GFX7-NEXT: s_mov_b32 s4, s8
4984 ; GFX7-NEXT: s_mov_b32 s5, s9
4985 ; GFX7-NEXT: s_mov_b32 s9, s10
4986 ; GFX7-NEXT: s_mov_b32 s8, s11
4987 ; GFX7-NEXT: s_add_u32 s4, s4, s9
4988 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
4989 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
4990 ; GFX7-NEXT: s_mov_b32 s5, s8
4991 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
4992 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
4993 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
4994 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
4995 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
4996 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
4997 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4998 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
4999 ; GFX7-NEXT: s_endpgm
5000 ;
5001 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5002 ; GFX10-WGP: ; %bb.0: ; %entry
5003 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
5004 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
5005 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5006 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
5007 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
5008 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5009 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
5010 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
5011 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5012 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
5013 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5014 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5015 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5016 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5017 ; GFX10-WGP-NEXT: buffer_gl0_inv
5018 ; GFX10-WGP-NEXT: s_endpgm
5019 ;
5020 ; GFX10-CU-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5021 ; GFX10-CU: ; %bb.0: ; %entry
5022 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
5023 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
5024 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5025 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
5026 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
5027 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5028 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
5029 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
5030 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5031 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
5032 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5033 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5034 ; GFX10-CU-NEXT: s_endpgm
5035 ;
5036 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5037 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5038 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
5039 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
5040 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
5041 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5042 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
5043 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
5044 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
5045 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
5046 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
5047 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
5048 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
5049 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
5050 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
5051 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
5052 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5053 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
5054 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5055 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5056 ; SKIP-CACHE-INV-NEXT: s_endpgm
5057 ;
5058 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5059 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5060 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5061 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5062 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5063 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5064 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5065 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5066 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5067 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5068 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5069 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5070 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5071 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5072 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5073 ;
5074 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5075 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5076 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5077 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5078 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5079 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5080 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5081 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5082 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5083 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5084 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5085 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5086 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5087 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5088 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5089 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5090 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5091 ;
5092 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5093 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5094 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5095 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5096 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5097 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5098 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5099 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5100 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5101 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5102 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5103 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5104 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5105 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5106 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5107 ;
5108 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5109 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5110 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5111 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5112 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5113 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5114 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5115 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5116 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5117 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5118 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5119 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5120 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5121 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5122 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5123 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5124 ; GFX940-TGSPLIT-NEXT: s_endpgm
5125 ;
5126 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5127 ; GFX11-WGP: ; %bb.0: ; %entry
5128 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5129 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
5130 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5131 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5132 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5133 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5134 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
5135 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
5136 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5137 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
5138 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5139 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5140 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5141 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5142 ; GFX11-WGP-NEXT: buffer_gl0_inv
5143 ; GFX11-WGP-NEXT: s_endpgm
5144 ;
5145 ; GFX11-CU-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5146 ; GFX11-CU: ; %bb.0: ; %entry
5147 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5148 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
5149 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5150 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5151 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5152 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5153 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
5154 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
5155 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5156 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
5157 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5158 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5159 ; GFX11-CU-NEXT: s_endpgm
5160 ;
5161 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5162 ; GFX12-WGP: ; %bb.0: ; %entry
5163 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5164 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
5165 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5166 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5167 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5168 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5169 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
5170 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
5171 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5172 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
5173 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5174 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5175 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5176 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5177 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5178 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
5179 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5180 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5181 ; GFX12-WGP-NEXT: s_endpgm
5182 ;
5183 ; GFX12-CU-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
5184 ; GFX12-CU: ; %bb.0: ; %entry
5185 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5186 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
5187 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5188 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5189 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5190 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5191 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
5192 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
5193 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5194 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
5195 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5196 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5197 ; GFX12-CU-NEXT: s_endpgm
5198 ptr addrspace(1) %out, i32 %in, i32 %old) {
5199 entry:
5200 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
5201 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
5202 ret void
5203 }
5205 define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
5206 ; GFX6-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5207 ; GFX6: ; %bb.0: ; %entry
5208 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5209 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
5210 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
5211 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5212 ; GFX6-NEXT: s_mov_b32 s12, s5
5213 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
5214 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
5215 ; GFX6-NEXT: s_mov_b32 s11, -1
5216 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
5217 ; GFX6-NEXT: s_mov_b32 s5, s12
5218 ; GFX6-NEXT: s_mov_b32 s6, s11
5219 ; GFX6-NEXT: s_mov_b32 s7, s10
5220 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
5221 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
5222 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5223 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
5224 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5225 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5226 ; GFX6-NEXT: s_endpgm
5227 ;
5228 ; GFX7-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5229 ; GFX7: ; %bb.0: ; %entry
5230 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5231 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5232 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5233 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5234 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5235 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5236 ; GFX7-NEXT: s_mov_b32 s4, s8
5237 ; GFX7-NEXT: s_mov_b32 s5, s9
5238 ; GFX7-NEXT: s_mov_b32 s9, s10
5239 ; GFX7-NEXT: s_mov_b32 s8, s11
5240 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5241 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5242 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5243 ; GFX7-NEXT: s_mov_b32 s5, s8
5244 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5245 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5246 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5247 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5248 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5249 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5250 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5251 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5252 ; GFX7-NEXT: s_endpgm
5253 ;
5254 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5255 ; GFX10-WGP: ; %bb.0: ; %entry
5256 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
5257 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
5258 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5259 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
5260 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
5261 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5262 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
5263 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
5264 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5265 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
5266 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5267 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5268 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5269 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5270 ; GFX10-WGP-NEXT: buffer_gl0_inv
5271 ; GFX10-WGP-NEXT: s_endpgm
5272 ;
5273 ; GFX10-CU-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5274 ; GFX10-CU: ; %bb.0: ; %entry
5275 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
5276 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
5277 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5278 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
5279 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
5280 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5281 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
5282 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
5283 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5284 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
5285 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5286 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5287 ; GFX10-CU-NEXT: s_endpgm
5288 ;
5289 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5290 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5291 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
5292 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
5293 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
5294 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5295 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
5296 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
5297 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
5298 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
5299 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
5300 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
5301 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
5302 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
5303 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
5304 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
5305 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
5307 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5308 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5309 ; SKIP-CACHE-INV-NEXT: s_endpgm
5310 ;
5311 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5312 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5313 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5314 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5315 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5316 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5317 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5318 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5319 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5320 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5321 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5322 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5323 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5324 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5325 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5326 ;
5327 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5328 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5329 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5330 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5331 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5332 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5333 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5334 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5335 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5336 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5337 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5338 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5339 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5340 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5341 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5342 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5343 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5344 ;
5345 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5346 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5347 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5348 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5349 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5350 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5351 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5352 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5353 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5354 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5355 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5357 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5358 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5359 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5360 ;
5361 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5362 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5363 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5364 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5365 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5366 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5367 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5368 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5369 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5370 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5371 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5372 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5373 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5374 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5375 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5376 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5377 ; GFX940-TGSPLIT-NEXT: s_endpgm
5378 ;
5379 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5380 ; GFX11-WGP: ; %bb.0: ; %entry
5381 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5382 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
5383 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5384 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5385 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5386 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5387 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
5388 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
5389 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5390 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
5391 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5392 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5393 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5394 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5395 ; GFX11-WGP-NEXT: buffer_gl0_inv
5396 ; GFX11-WGP-NEXT: s_endpgm
5397 ;
5398 ; GFX11-CU-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5399 ; GFX11-CU: ; %bb.0: ; %entry
5400 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5401 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
5402 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5403 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5404 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5405 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5406 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
5407 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
5408 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5409 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
5410 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5411 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5412 ; GFX11-CU-NEXT: s_endpgm
5413 ;
5414 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5415 ; GFX12-WGP: ; %bb.0: ; %entry
5416 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5417 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
5418 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5419 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5420 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5421 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
5423 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
5424 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5425 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
5426 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5427 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5428 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5429 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5430 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5431 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
5432 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5433 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5434 ; GFX12-WGP-NEXT: s_endpgm
5435 ;
5436 ; GFX12-CU-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
5437 ; GFX12-CU: ; %bb.0: ; %entry
5438 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5439 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
5440 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5441 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5442 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5443 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5444 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
5445 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
5446 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5447 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
5448 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5449 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5450 ; GFX12-CU-NEXT: s_endpgm
5451 ptr addrspace(1) %out, i32 %in, i32 %old) {
5452 entry:
5453 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
5454 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
5455 ret void
5456 }
5458 define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_cmpxchg(
5459 ; GFX6-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5460 ; GFX6: ; %bb.0: ; %entry
5461 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5462 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
5463 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
5464 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5465 ; GFX6-NEXT: s_mov_b32 s12, s5
5466 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
5467 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
5468 ; GFX6-NEXT: s_mov_b32 s11, -1
5469 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
5470 ; GFX6-NEXT: s_mov_b32 s5, s12
5471 ; GFX6-NEXT: s_mov_b32 s6, s11
5472 ; GFX6-NEXT: s_mov_b32 s7, s10
5473 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
5474 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
5475 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5476 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
5477 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5478 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5479 ; GFX6-NEXT: s_endpgm
5480 ;
5481 ; GFX7-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5482 ; GFX7: ; %bb.0: ; %entry
5483 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5484 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5485 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5486 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5487 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5488 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5489 ; GFX7-NEXT: s_mov_b32 s4, s8
5490 ; GFX7-NEXT: s_mov_b32 s5, s9
5491 ; GFX7-NEXT: s_mov_b32 s9, s10
5492 ; GFX7-NEXT: s_mov_b32 s8, s11
5493 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5494 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5495 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5496 ; GFX7-NEXT: s_mov_b32 s5, s8
5497 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5498 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5499 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5500 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5501 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5502 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5503 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5504 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5505 ; GFX7-NEXT: s_endpgm
5506 ;
5507 ; GFX10-WGP-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5508 ; GFX10-WGP: ; %bb.0: ; %entry
5509 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
5510 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
5511 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5512 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
5513 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
5514 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5515 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
5516 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
5517 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5518 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
5519 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5520 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5521 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5522 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5523 ; GFX10-WGP-NEXT: buffer_gl0_inv
5524 ; GFX10-WGP-NEXT: s_endpgm
5525 ;
5526 ; GFX10-CU-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5527 ; GFX10-CU: ; %bb.0: ; %entry
5528 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
5529 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
5530 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5531 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
5532 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
5533 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5534 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
5535 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
5536 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5537 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
5538 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5539 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5540 ; GFX10-CU-NEXT: s_endpgm
5541 ;
5542 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5543 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5544 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
5545 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
5546 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
5547 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5548 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
5549 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
5550 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
5551 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
5552 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
5553 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
5554 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
5555 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
5556 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
5557 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
5558 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5559 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
5560 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5561 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5562 ; SKIP-CACHE-INV-NEXT: s_endpgm
5563 ;
5564 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5565 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5566 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5567 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5568 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5569 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5570 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5571 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5572 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5573 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5574 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5575 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5576 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5577 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5578 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5579 ;
5580 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5581 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5582 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5583 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5584 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5585 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5586 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5587 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5588 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5589 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5590 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5591 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5592 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5593 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5594 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5595 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5596 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5597 ;
5598 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5599 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5600 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5601 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5602 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5603 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5604 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5605 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5606 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5607 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5608 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5609 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5610 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5611 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5612 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5613 ;
5614 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5615 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5616 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5617 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5618 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5619 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5620 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5621 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5622 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5623 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5624 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5625 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5626 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5627 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5628 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5629 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5630 ; GFX940-TGSPLIT-NEXT: s_endpgm
5631 ;
5632 ; GFX11-WGP-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5633 ; GFX11-WGP: ; %bb.0: ; %entry
5634 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5635 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
5636 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5637 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5638 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5639 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5640 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
5641 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
5642 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5643 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
5644 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5645 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5646 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5647 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5648 ; GFX11-WGP-NEXT: buffer_gl0_inv
5649 ; GFX11-WGP-NEXT: s_endpgm
5650 ;
5651 ; GFX11-CU-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5652 ; GFX11-CU: ; %bb.0: ; %entry
5653 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5654 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
5655 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5656 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5657 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5658 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5659 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
5660 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
5661 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5662 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
5663 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5664 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5665 ; GFX11-CU-NEXT: s_endpgm
5666 ;
5667 ; GFX12-WGP-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5668 ; GFX12-WGP: ; %bb.0: ; %entry
5669 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5670 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
5671 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5672 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5673 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5674 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5675 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
5676 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
5677 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5678 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
5679 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5680 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5681 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5682 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5683 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5684 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
5685 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5686 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5687 ; GFX12-WGP-NEXT: s_endpgm
5688 ;
5689 ; GFX12-CU-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
5690 ; GFX12-CU: ; %bb.0: ; %entry
5691 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5692 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
5693 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5694 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5695 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5696 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5697 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
5698 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
5699 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5700 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
5701 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5702 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5703 ; GFX12-CU-NEXT: s_endpgm
5704 ptr addrspace(1) %out, i32 %in, i32 %old) {
5705 entry:
5706 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
5707 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
5708 ret void
5709 }
5711 define amdgpu_kernel void @global_workgroup_acquire_seq_cst_cmpxchg(
5712 ; GFX6-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5713 ; GFX6: ; %bb.0: ; %entry
5714 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5715 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
5716 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
5717 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5718 ; GFX6-NEXT: s_mov_b32 s12, s5
5719 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
5720 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
5721 ; GFX6-NEXT: s_mov_b32 s11, -1
5722 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
5723 ; GFX6-NEXT: s_mov_b32 s5, s12
5724 ; GFX6-NEXT: s_mov_b32 s6, s11
5725 ; GFX6-NEXT: s_mov_b32 s7, s10
5726 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
5727 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
5728 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5729 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
5730 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5731 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5732 ; GFX6-NEXT: s_endpgm
5733 ;
5734 ; GFX7-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5735 ; GFX7: ; %bb.0: ; %entry
5736 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5737 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5738 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5739 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5740 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5741 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5742 ; GFX7-NEXT: s_mov_b32 s4, s8
5743 ; GFX7-NEXT: s_mov_b32 s5, s9
5744 ; GFX7-NEXT: s_mov_b32 s9, s10
5745 ; GFX7-NEXT: s_mov_b32 s8, s11
5746 ; GFX7-NEXT: s_add_u32 s4, s4, s9
5747 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
5748 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
5749 ; GFX7-NEXT: s_mov_b32 s5, s8
5750 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
5751 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
5752 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5753 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
5754 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
5755 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
5756 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5757 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
5758 ; GFX7-NEXT: s_endpgm
5759 ;
5760 ; GFX10-WGP-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5761 ; GFX10-WGP: ; %bb.0: ; %entry
5762 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
5763 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
5764 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5765 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
5766 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
5767 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
5768 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
5769 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
5770 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5771 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
5772 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5773 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5774 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5775 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5776 ; GFX10-WGP-NEXT: buffer_gl0_inv
5777 ; GFX10-WGP-NEXT: s_endpgm
5778 ;
5779 ; GFX10-CU-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5780 ; GFX10-CU: ; %bb.0: ; %entry
5781 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
5782 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
5783 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5784 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
5785 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
5786 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5787 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
5788 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
5789 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5790 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
5791 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
5792 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
5793 ; GFX10-CU-NEXT: s_endpgm
5794 ;
5795 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5796 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
5797 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
5798 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
5799 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
5800 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5801 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
5802 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
5803 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
5804 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
5805 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
5806 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
5807 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
5808 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
5809 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
5810 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
5811 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5812 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
5813 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
5814 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
5815 ; SKIP-CACHE-INV-NEXT: s_endpgm
5816 ;
5817 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5818 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
5819 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5820 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5821 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5822 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5823 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5824 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5825 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5826 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5827 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5828 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5829 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5830 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5831 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
5832 ;
5833 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5834 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
5835 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
5836 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5837 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
5838 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
5839 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
5840 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5841 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
5842 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
5843 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5844 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5845 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5846 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
5847 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5848 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
5849 ; GFX90A-TGSPLIT-NEXT: s_endpgm
5850 ;
5851 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5852 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
5853 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5854 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5855 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5856 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5857 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5858 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5859 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5860 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5861 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5862 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5863 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5864 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5865 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
5866 ;
5867 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5868 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
5869 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
5870 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
5871 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
5872 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
5873 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
5874 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
5875 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
5876 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
5877 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
5878 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
5879 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5880 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
5881 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
5882 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
5883 ; GFX940-TGSPLIT-NEXT: s_endpgm
5884 ;
5885 ; GFX11-WGP-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5886 ; GFX11-WGP: ; %bb.0: ; %entry
5887 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5888 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
5889 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5890 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5891 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5892 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
5893 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
5894 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
5895 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5896 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
5897 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
5898 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5899 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5900 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
5901 ; GFX11-WGP-NEXT: buffer_gl0_inv
5902 ; GFX11-WGP-NEXT: s_endpgm
5903 ;
5904 ; GFX11-CU-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5905 ; GFX11-CU: ; %bb.0: ; %entry
5906 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5907 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
5908 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5909 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5910 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5911 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5912 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
5913 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
5914 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5915 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
5916 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
5917 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5918 ; GFX11-CU-NEXT: s_endpgm
5919 ;
5920 ; GFX12-WGP-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5921 ; GFX12-WGP: ; %bb.0: ; %entry
5922 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
5923 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
5924 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5925 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
5926 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
5927 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
5928 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
5929 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
5930 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5931 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
5932 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
5933 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
5934 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
5935 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5936 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
5937 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
5938 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
5939 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
5940 ; GFX12-WGP-NEXT: s_endpgm
5941 ;
5942 ; GFX12-CU-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
5943 ; GFX12-CU: ; %bb.0: ; %entry
5944 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
5945 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
5946 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
5947 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
5948 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
5949 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
5950 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
5951 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
5952 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
5953 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
5954 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
5955 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
5956 ; GFX12-CU-NEXT: s_endpgm
5957 ptr addrspace(1) %out, i32 %in, i32 %old) {
5958 entry:
5959 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
5960 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
5961 ret void
5962 }
5964 define amdgpu_kernel void @global_workgroup_release_seq_cst_cmpxchg(
5965 ; GFX6-LABEL: global_workgroup_release_seq_cst_cmpxchg:
5966 ; GFX6: ; %bb.0: ; %entry
5967 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
5968 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
5969 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
5970 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5971 ; GFX6-NEXT: s_mov_b32 s12, s5
5972 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
5973 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
5974 ; GFX6-NEXT: s_mov_b32 s11, -1
5975 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
5976 ; GFX6-NEXT: s_mov_b32 s5, s12
5977 ; GFX6-NEXT: s_mov_b32 s6, s11
5978 ; GFX6-NEXT: s_mov_b32 s7, s10
5979 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
5980 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
5981 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
5982 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
5983 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
5984 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
5985 ; GFX6-NEXT: s_endpgm
5986 ;
5987 ; GFX7-LABEL: global_workgroup_release_seq_cst_cmpxchg:
5988 ; GFX7: ; %bb.0: ; %entry
5989 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
5990 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
5991 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
5992 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
5993 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
5994 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
5995 ; GFX7-NEXT: s_mov_b32 s4, s8
5996 ; GFX7-NEXT: s_mov_b32 s5, s9
5997 ; GFX7-NEXT: s_mov_b32 s9, s10
5998 ; GFX7-NEXT: s_mov_b32 s8, s11
5999 ; GFX7-NEXT: s_add_u32 s4, s4, s9
6000 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
6001 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6002 ; GFX7-NEXT: s_mov_b32 s5, s8
6003 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
6004 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6005 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6006 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6007 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6008 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6009 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6010 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6011 ; GFX7-NEXT: s_endpgm
6012 ;
6013 ; GFX10-WGP-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6014 ; GFX10-WGP: ; %bb.0: ; %entry
6015 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
6016 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
6017 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6018 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
6019 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
6020 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6021 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6022 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
6023 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6024 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
6025 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6026 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6027 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6028 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6029 ; GFX10-WGP-NEXT: buffer_gl0_inv
6030 ; GFX10-WGP-NEXT: s_endpgm
6031 ;
6032 ; GFX10-CU-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6033 ; GFX10-CU: ; %bb.0: ; %entry
6034 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
6035 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
6036 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6037 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
6038 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
6039 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6040 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6041 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
6042 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6043 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
6044 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6045 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6046 ; GFX10-CU-NEXT: s_endpgm
6047 ;
6048 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6049 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6050 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6051 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6052 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6053 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6054 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
6055 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
6056 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
6057 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
6058 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
6059 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
6060 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
6061 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6062 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
6063 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
6064 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6065 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
6066 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6067 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6068 ; SKIP-CACHE-INV-NEXT: s_endpgm
6069 ;
6070 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6071 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6072 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6073 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6074 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6075 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6076 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6077 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6078 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6079 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6080 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6081 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6082 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6083 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6084 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6085 ;
6086 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6087 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6088 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6089 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6090 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6091 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6092 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6093 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6094 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6095 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6096 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6097 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6098 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6099 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6100 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6101 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6102 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6103 ;
6104 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6105 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6106 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6107 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6108 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6109 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6110 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6111 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6112 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6113 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6114 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6115 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6116 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6117 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6118 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6119 ;
6120 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6121 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6122 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6123 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6124 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6125 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6126 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6127 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6128 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6129 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6130 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6131 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6132 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6133 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6134 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6135 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
6136 ; GFX940-TGSPLIT-NEXT: s_endpgm
6137 ;
6138 ; GFX11-WGP-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6139 ; GFX11-WGP: ; %bb.0: ; %entry
6140 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6141 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
6142 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6143 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6144 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6145 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6146 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
6147 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
6148 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6149 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
6150 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6151 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6152 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6153 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6154 ; GFX11-WGP-NEXT: buffer_gl0_inv
6155 ; GFX11-WGP-NEXT: s_endpgm
6156 ;
6157 ; GFX11-CU-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6158 ; GFX11-CU: ; %bb.0: ; %entry
6159 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6160 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
6161 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6162 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6163 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6164 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6165 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
6166 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
6167 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6168 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
6169 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6170 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6171 ; GFX11-CU-NEXT: s_endpgm
6172 ;
6173 ; GFX12-WGP-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6174 ; GFX12-WGP: ; %bb.0: ; %entry
6175 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6176 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
6177 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6178 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6179 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6180 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6181 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
6182 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
6183 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6184 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
6185 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
6186 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6187 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6188 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6189 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6190 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
6191 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6192 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
6193 ; GFX12-WGP-NEXT: s_endpgm
6194 ;
6195 ; GFX12-CU-LABEL: global_workgroup_release_seq_cst_cmpxchg:
6196 ; GFX12-CU: ; %bb.0: ; %entry
6197 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6198 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
6199 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6200 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6201 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6202 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6203 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
6204 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
6205 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6206 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
6207 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6208 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6209 ; GFX12-CU-NEXT: s_endpgm
6210 ptr addrspace(1) %out, i32 %in, i32 %old) {
6211 entry:
6212 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
6213 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
6214 ret void
6215 }
6217 define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_cmpxchg(
6218 ; GFX6-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6219 ; GFX6: ; %bb.0: ; %entry
6220 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6221 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
6222 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
6223 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6224 ; GFX6-NEXT: s_mov_b32 s12, s5
6225 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
6226 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
6227 ; GFX6-NEXT: s_mov_b32 s11, -1
6228 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
6229 ; GFX6-NEXT: s_mov_b32 s5, s12
6230 ; GFX6-NEXT: s_mov_b32 s6, s11
6231 ; GFX6-NEXT: s_mov_b32 s7, s10
6232 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
6233 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
6234 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6235 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
6236 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6237 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6238 ; GFX6-NEXT: s_endpgm
6239 ;
6240 ; GFX7-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6241 ; GFX7: ; %bb.0: ; %entry
6242 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
6243 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6244 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
6245 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
6246 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
6247 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6248 ; GFX7-NEXT: s_mov_b32 s4, s8
6249 ; GFX7-NEXT: s_mov_b32 s5, s9
6250 ; GFX7-NEXT: s_mov_b32 s9, s10
6251 ; GFX7-NEXT: s_mov_b32 s8, s11
6252 ; GFX7-NEXT: s_add_u32 s4, s4, s9
6253 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
6254 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6255 ; GFX7-NEXT: s_mov_b32 s5, s8
6256 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
6257 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6258 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6259 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6260 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6261 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6262 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6263 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6264 ; GFX7-NEXT: s_endpgm
6265 ;
6266 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6267 ; GFX10-WGP: ; %bb.0: ; %entry
6268 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
6269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
6270 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6271 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
6272 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
6273 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6274 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6275 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
6276 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6277 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
6278 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6279 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6280 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6281 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6282 ; GFX10-WGP-NEXT: buffer_gl0_inv
6283 ; GFX10-WGP-NEXT: s_endpgm
6284 ;
6285 ; GFX10-CU-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6286 ; GFX10-CU: ; %bb.0: ; %entry
6287 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
6288 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
6289 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6290 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
6291 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
6292 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6293 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6294 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
6295 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6296 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
6297 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6298 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6299 ; GFX10-CU-NEXT: s_endpgm
6300 ;
6301 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6302 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6303 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6304 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6305 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6306 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6307 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
6308 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
6309 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
6310 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
6311 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
6312 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
6313 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
6314 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6315 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
6316 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
6317 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6318 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
6319 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6320 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6321 ; SKIP-CACHE-INV-NEXT: s_endpgm
6322 ;
6323 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6324 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6325 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6326 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6327 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6328 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6329 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6330 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6331 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6332 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6333 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6334 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6335 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6336 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6337 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6338 ;
6339 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6340 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6341 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6342 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6343 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6344 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6345 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6346 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6347 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6348 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6349 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6350 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6351 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6352 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6353 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6354 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6355 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6356 ;
6357 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6358 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6359 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6360 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6361 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6362 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6363 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6364 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6365 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6366 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6367 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6368 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6369 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6370 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6371 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6372 ;
6373 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6374 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6375 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6376 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6377 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6378 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6379 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6380 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6381 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6382 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6383 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6384 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6385 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6386 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6387 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6388 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
6389 ; GFX940-TGSPLIT-NEXT: s_endpgm
6390 ;
6391 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6392 ; GFX11-WGP: ; %bb.0: ; %entry
6393 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6394 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
6395 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6396 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6397 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6398 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6399 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
6400 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
6401 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6402 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
6403 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6404 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6405 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6406 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6407 ; GFX11-WGP-NEXT: buffer_gl0_inv
6408 ; GFX11-WGP-NEXT: s_endpgm
6409 ;
6410 ; GFX11-CU-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6411 ; GFX11-CU: ; %bb.0: ; %entry
6412 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6413 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
6414 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6415 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6416 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6417 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6418 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
6419 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
6420 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6421 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
6422 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6423 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6424 ; GFX11-CU-NEXT: s_endpgm
6425 ;
6426 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6427 ; GFX12-WGP: ; %bb.0: ; %entry
6428 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6429 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
6430 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6431 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6432 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6433 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6434 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
6435 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
6436 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6437 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
6438 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
6439 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6440 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6441 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6442 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6443 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
6444 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6445 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
6446 ; GFX12-WGP-NEXT: s_endpgm
6447 ;
6448 ; GFX12-CU-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
6449 ; GFX12-CU: ; %bb.0: ; %entry
6450 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6451 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
6452 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6453 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6454 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6455 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6456 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
6457 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
6458 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6459 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
6460 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6461 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6462 ; GFX12-CU-NEXT: s_endpgm
6463 ptr addrspace(1) %out, i32 %in, i32 %old) {
6464 entry:
6465 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
6466 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
6467 ret void
6468 }
6470 define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
6471 ; GFX6-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6472 ; GFX6: ; %bb.0: ; %entry
6473 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6474 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
6475 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
6476 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6477 ; GFX6-NEXT: s_mov_b32 s12, s5
6478 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
6479 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
6480 ; GFX6-NEXT: s_mov_b32 s11, -1
6481 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
6482 ; GFX6-NEXT: s_mov_b32 s5, s12
6483 ; GFX6-NEXT: s_mov_b32 s6, s11
6484 ; GFX6-NEXT: s_mov_b32 s7, s10
6485 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
6486 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
6487 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6488 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
6489 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6490 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6491 ; GFX6-NEXT: s_endpgm
6492 ;
6493 ; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6494 ; GFX7: ; %bb.0: ; %entry
6495 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
6496 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
6497 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
6498 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
6499 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
6500 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6501 ; GFX7-NEXT: s_mov_b32 s4, s8
6502 ; GFX7-NEXT: s_mov_b32 s5, s9
6503 ; GFX7-NEXT: s_mov_b32 s9, s10
6504 ; GFX7-NEXT: s_mov_b32 s8, s11
6505 ; GFX7-NEXT: s_add_u32 s4, s4, s9
6506 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
6507 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
6508 ; GFX7-NEXT: s_mov_b32 s5, s8
6509 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
6510 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6511 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6512 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6513 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6514 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6515 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6516 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
6517 ; GFX7-NEXT: s_endpgm
6518 ;
6519 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6520 ; GFX10-WGP: ; %bb.0: ; %entry
6521 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
6522 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
6523 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6524 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
6525 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
6526 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6527 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6528 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
6529 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6530 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
6531 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6532 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6533 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6534 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6535 ; GFX10-WGP-NEXT: buffer_gl0_inv
6536 ; GFX10-WGP-NEXT: s_endpgm
6537 ;
6538 ; GFX10-CU-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6539 ; GFX10-CU: ; %bb.0: ; %entry
6540 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
6541 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
6542 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6543 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
6544 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
6545 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6546 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6547 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
6548 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6549 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
6550 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6551 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
6552 ; GFX10-CU-NEXT: s_endpgm
6553 ;
6554 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6555 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6556 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6557 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6558 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6559 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6560 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
6561 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
6562 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
6563 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
6564 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
6565 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
6566 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
6567 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6568 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
6569 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
6570 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6571 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
6572 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6573 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6574 ; SKIP-CACHE-INV-NEXT: s_endpgm
6575 ;
6576 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6577 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6578 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6579 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6580 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6581 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6582 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6583 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6584 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6585 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6586 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6587 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6588 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6589 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6590 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6591 ;
6592 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6593 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6594 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6595 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6596 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6597 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6598 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6599 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6600 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6601 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6602 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6603 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6604 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6605 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
6606 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6607 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
6608 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6609 ;
6610 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6611 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6612 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6613 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6614 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6615 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6616 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6617 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6618 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6619 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6620 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6621 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6622 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6623 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6624 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6625 ;
6626 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6627 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6628 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6629 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6630 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6631 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6632 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6633 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6634 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6635 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6636 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6637 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6638 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6639 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
6640 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6641 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
6642 ; GFX940-TGSPLIT-NEXT: s_endpgm
6643 ;
6644 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6645 ; GFX11-WGP: ; %bb.0: ; %entry
6646 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6647 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
6648 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6649 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6650 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6651 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6652 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
6653 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
6654 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6655 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
6656 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
6657 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6658 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6659 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
6660 ; GFX11-WGP-NEXT: buffer_gl0_inv
6661 ; GFX11-WGP-NEXT: s_endpgm
6662 ;
6663 ; GFX11-CU-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6664 ; GFX11-CU: ; %bb.0: ; %entry
6665 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6666 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
6667 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6668 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6669 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6670 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6671 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
6672 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
6673 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6674 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
6675 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6676 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6677 ; GFX11-CU-NEXT: s_endpgm
6678 ;
6679 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6680 ; GFX12-WGP: ; %bb.0: ; %entry
6681 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6682 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
6683 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6684 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6685 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6686 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6687 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
6688 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
6689 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6690 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
6691 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
6692 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
6693 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
6694 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6695 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
6696 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
6697 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
6698 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
6699 ; GFX12-WGP-NEXT: s_endpgm
6700 ;
6701 ; GFX12-CU-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
6702 ; GFX12-CU: ; %bb.0: ; %entry
6703 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6704 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
6705 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6706 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6707 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6708 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6709 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
6710 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
6711 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6712 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
6713 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
6714 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
6715 ; GFX12-CU-NEXT: s_endpgm
6716 ptr addrspace(1) %out, i32 %in, i32 %old) {
6717 entry:
6718 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
6719 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
6720 ret void
6721 }
6723 define amdgpu_kernel void @global_workgroup_monotonic_monotonic_ret_cmpxchg(
6724 ; GFX6-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6725 ; GFX6: ; %bb.0: ; %entry
6726 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6727 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
6728 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
6729 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6730 ; GFX6-NEXT: s_mov_b32 s12, s5
6731 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
6732 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
6733 ; GFX6-NEXT: s_mov_b32 s11, -1
6734 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
6735 ; GFX6-NEXT: s_mov_b32 s5, s12
6736 ; GFX6-NEXT: s_mov_b32 s6, s11
6737 ; GFX6-NEXT: s_mov_b32 s7, s10
6738 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
6739 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
6740 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6741 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
6742 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6743 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
6744 ; GFX6-NEXT: s_waitcnt vmcnt(0)
6745 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
6746 ; GFX6-NEXT: s_endpgm
6747 ;
6748 ; GFX7-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6749 ; GFX7: ; %bb.0: ; %entry
6750 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6751 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
6752 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
6753 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
6754 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
6755 ; GFX7-NEXT: s_mov_b32 s6, s4
6756 ; GFX7-NEXT: s_mov_b32 s7, s5
6757 ; GFX7-NEXT: s_mov_b32 s11, s12
6758 ; GFX7-NEXT: s_mov_b32 s10, s13
6759 ; GFX7-NEXT: s_add_u32 s6, s6, s11
6760 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
6761 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
6762 ; GFX7-NEXT: s_mov_b32 s7, s10
6763 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
6764 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
6765 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6766 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
6767 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
6768 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
6769 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
6770 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
6771 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
6772 ; GFX7-NEXT: s_waitcnt vmcnt(0)
6773 ; GFX7-NEXT: flat_store_dword v[0:1], v2
6774 ; GFX7-NEXT: s_endpgm
6775 ;
6776 ; GFX10-WGP-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6777 ; GFX10-WGP: ; %bb.0: ; %entry
6778 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
6779 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
6780 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6781 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
6782 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
6783 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
6784 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
6785 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
6786 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
6788 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
6789 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
6790 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
6791 ; GFX10-WGP-NEXT: s_endpgm
6792 ;
6793 ; GFX10-CU-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6794 ; GFX10-CU: ; %bb.0: ; %entry
6795 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
6796 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
6797 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6798 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
6799 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
6800 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
6801 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
6802 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
6803 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6804 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
6805 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
6806 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
6807 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
6808 ; GFX10-CU-NEXT: s_endpgm
6809 ;
6810 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6811 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
6812 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
6813 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
6814 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
6815 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
6816 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
6817 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
6818 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
6819 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
6820 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
6821 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
6822 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
6823 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
6824 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
6825 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
6826 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6827 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
6828 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
6829 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
6830 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
6831 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
6832 ; SKIP-CACHE-INV-NEXT: s_endpgm
6833 ;
6834 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6835 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
6836 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6837 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6838 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6839 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6840 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6841 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6842 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6843 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6844 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6845 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6846 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
6847 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6848 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
6849 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
6850 ;
6851 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6852 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
6853 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
6854 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6855 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
6856 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
6857 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
6858 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6859 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
6860 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
6861 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6862 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6863 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
6864 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6865 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
6866 ; GFX90A-TGSPLIT-NEXT: s_endpgm
6867 ;
6868 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6869 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
6870 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6871 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6872 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6873 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6874 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6875 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6876 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6877 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6878 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6879 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6880 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
6881 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
6882 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
6883 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
6884 ;
6885 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6886 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
6887 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
6888 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
6889 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
6890 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
6891 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
6892 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
6893 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
6894 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
6895 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
6896 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
6897 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
6898 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
6899 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
6900 ; GFX940-TGSPLIT-NEXT: s_endpgm
6901 ;
6902 ; GFX11-WGP-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6903 ; GFX11-WGP: ; %bb.0: ; %entry
6904 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6905 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
6906 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6907 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6908 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6909 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
6910 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
6911 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
6912 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6913 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
6914 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
6915 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
6916 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
6917 ; GFX11-WGP-NEXT: s_endpgm
6918 ;
6919 ; GFX11-CU-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6920 ; GFX11-CU: ; %bb.0: ; %entry
6921 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6922 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
6923 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6924 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6925 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6926 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
6927 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
6928 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
6929 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6930 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
6931 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
6932 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
6933 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
6934 ; GFX11-CU-NEXT: s_endpgm
6935 ;
6936 ; GFX12-WGP-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6937 ; GFX12-WGP: ; %bb.0: ; %entry
6938 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
6939 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
6940 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6941 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
6942 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
6943 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
6944 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
6945 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
6946 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6947 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
6948 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
6949 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
6950 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
6951 ; GFX12-WGP-NEXT: s_endpgm
6952 ;
6953 ; GFX12-CU-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
6954 ; GFX12-CU: ; %bb.0: ; %entry
6955 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
6956 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
6957 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
6958 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
6959 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
6960 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
6961 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
6962 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
6963 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
6964 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
6965 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
6966 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
6967 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
6968 ; GFX12-CU-NEXT: s_endpgm
6969 ptr addrspace(1) %out, i32 %in, i32 %old) {
6970 entry:
6971 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
6972 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
6973 %val0 = extractvalue { i32, i1 } %val, 0
6974 store i32 %val0, ptr addrspace(1) %out, align 4
6975 ret void
6976 }
6978 define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
6979 ; GFX6-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
6980 ; GFX6: ; %bb.0: ; %entry
6981 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
6982 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
6983 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
6984 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
6985 ; GFX6-NEXT: s_mov_b32 s12, s5
6986 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
6987 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
6988 ; GFX6-NEXT: s_mov_b32 s11, -1
6989 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
6990 ; GFX6-NEXT: s_mov_b32 s5, s12
6991 ; GFX6-NEXT: s_mov_b32 s6, s11
6992 ; GFX6-NEXT: s_mov_b32 s7, s10
6993 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
6994 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
6995 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
6996 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
6997 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
6998 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
6999 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7000 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
7001 ; GFX6-NEXT: s_endpgm
7002 ;
7003 ; GFX7-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7004 ; GFX7: ; %bb.0: ; %entry
7005 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7006 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7007 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7008 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7009 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7010 ; GFX7-NEXT: s_mov_b32 s6, s4
7011 ; GFX7-NEXT: s_mov_b32 s7, s5
7012 ; GFX7-NEXT: s_mov_b32 s11, s12
7013 ; GFX7-NEXT: s_mov_b32 s10, s13
7014 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7015 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7016 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7017 ; GFX7-NEXT: s_mov_b32 s7, s10
7018 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7019 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7020 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7021 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7022 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7023 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7024 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7025 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7026 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7027 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7028 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7029 ; GFX7-NEXT: s_endpgm
7030 ;
7031 ; GFX10-WGP-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7032 ; GFX10-WGP: ; %bb.0: ; %entry
7033 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
7034 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
7035 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7036 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
7037 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
7038 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7039 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7040 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
7041 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7042 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
7043 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7044 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
7045 ; GFX10-WGP-NEXT: buffer_gl0_inv
7046 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
7047 ; GFX10-WGP-NEXT: s_endpgm
7048 ;
7049 ; GFX10-CU-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7050 ; GFX10-CU: ; %bb.0: ; %entry
7051 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
7052 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
7053 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7054 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
7055 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
7056 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7057 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7058 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
7059 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7060 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
7061 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7062 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7063 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
7064 ; GFX10-CU-NEXT: s_endpgm
7065 ;
7066 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7067 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7068 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7069 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7070 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7071 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7072 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
7073 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
7074 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
7075 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
7076 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
7077 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
7078 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
7079 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7080 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
7081 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
7082 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7083 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
7084 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7085 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7086 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7087 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
7088 ; SKIP-CACHE-INV-NEXT: s_endpgm
7089 ;
7090 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7091 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7092 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7093 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7094 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7095 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7096 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7097 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7098 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7099 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7100 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7101 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7102 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7103 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7104 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7105 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7106 ;
7107 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7108 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7109 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7110 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7111 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7112 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7113 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7114 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7115 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7116 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7117 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7118 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7119 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7120 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7121 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7122 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7123 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7124 ;
7125 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7126 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7127 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7128 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7129 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7130 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7131 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7132 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7133 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7134 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7135 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7136 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7137 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7138 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7139 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7140 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7141 ;
7142 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7143 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7144 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7145 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7146 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7147 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7148 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7149 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7150 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7151 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7152 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7153 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7154 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7155 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7156 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7157 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7158 ; GFX940-TGSPLIT-NEXT: s_endpgm
7159 ;
7160 ; GFX11-WGP-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7161 ; GFX11-WGP: ; %bb.0: ; %entry
7162 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7163 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
7164 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7165 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7166 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7167 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7168 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
7169 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
7170 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7171 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
7172 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7173 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
7174 ; GFX11-WGP-NEXT: buffer_gl0_inv
7175 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7176 ; GFX11-WGP-NEXT: s_endpgm
7177 ;
7178 ; GFX11-CU-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7179 ; GFX11-CU: ; %bb.0: ; %entry
7180 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7181 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
7182 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7183 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7184 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7185 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7186 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
7187 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
7188 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7189 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
7190 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7191 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7192 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7193 ; GFX11-CU-NEXT: s_endpgm
7194 ;
7195 ; GFX12-WGP-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7196 ; GFX12-WGP: ; %bb.0: ; %entry
7197 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7198 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
7199 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7200 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7201 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7202 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7203 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
7204 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
7205 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7206 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
7207 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7208 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7209 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7210 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
7211 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
7212 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7213 ; GFX12-WGP-NEXT: s_endpgm
7214 ;
7215 ; GFX12-CU-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
7216 ; GFX12-CU: ; %bb.0: ; %entry
7217 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7218 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
7219 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7220 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7221 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7222 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7223 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
7224 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
7225 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7226 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
7227 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
7228 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7229 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7230 ; GFX12-CU-NEXT: s_endpgm
7231 ptr addrspace(1) %out, i32 %in, i32 %old) {
7232 entry:
7233 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
7234 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
7235 %val0 = extractvalue { i32, i1 } %val, 0
7236 store i32 %val0, ptr addrspace(1) %out, align 4
7237 ret void
7238 }
7240 define amdgpu_kernel void @global_workgroup_release_monotonic_ret_cmpxchg(
7241 ; GFX6-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7242 ; GFX6: ; %bb.0: ; %entry
7243 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7244 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
7245 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
7246 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7247 ; GFX6-NEXT: s_mov_b32 s12, s5
7248 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
7249 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
7250 ; GFX6-NEXT: s_mov_b32 s11, -1
7251 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
7252 ; GFX6-NEXT: s_mov_b32 s5, s12
7253 ; GFX6-NEXT: s_mov_b32 s6, s11
7254 ; GFX6-NEXT: s_mov_b32 s7, s10
7255 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
7256 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
7257 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7258 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
7259 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7260 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
7261 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7262 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7263 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
7264 ; GFX6-NEXT: s_endpgm
7265 ;
7266 ; GFX7-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7267 ; GFX7: ; %bb.0: ; %entry
7268 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7269 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7270 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7271 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7272 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7273 ; GFX7-NEXT: s_mov_b32 s6, s4
7274 ; GFX7-NEXT: s_mov_b32 s7, s5
7275 ; GFX7-NEXT: s_mov_b32 s11, s12
7276 ; GFX7-NEXT: s_mov_b32 s10, s13
7277 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7278 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7279 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7280 ; GFX7-NEXT: s_mov_b32 s7, s10
7281 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7282 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7283 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7284 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7285 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7286 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7287 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7288 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7289 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7290 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7291 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7292 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7293 ; GFX7-NEXT: s_endpgm
7294 ;
7295 ; GFX10-WGP-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7296 ; GFX10-WGP: ; %bb.0: ; %entry
7297 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
7298 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
7299 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7300 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
7301 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
7302 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7303 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7304 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
7305 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7306 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
7307 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7308 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7309 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7310 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
7311 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
7312 ; GFX10-WGP-NEXT: s_endpgm
7313 ;
7314 ; GFX10-CU-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7315 ; GFX10-CU: ; %bb.0: ; %entry
7316 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
7317 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
7318 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7319 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
7320 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
7321 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7322 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7323 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
7324 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7325 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
7326 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7327 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7328 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7329 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
7330 ; GFX10-CU-NEXT: s_endpgm
7331 ;
7332 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7333 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7334 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7335 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7336 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7337 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7338 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
7339 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
7340 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
7341 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
7342 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
7343 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
7344 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
7345 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7346 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
7347 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
7348 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7349 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
7350 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7351 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7352 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7353 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7354 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
7355 ; SKIP-CACHE-INV-NEXT: s_endpgm
7356 ;
7357 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7358 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7359 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7360 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7361 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7362 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7363 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7364 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7365 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7366 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7367 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7368 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7369 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7370 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7371 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7372 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7373 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7374 ;
7375 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7376 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7377 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7378 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7379 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7380 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7381 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7382 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7383 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7384 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7385 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7386 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7387 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7388 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7389 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7390 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7391 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7392 ;
7393 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7394 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7395 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7396 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7397 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7398 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7399 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7400 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7401 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7402 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7403 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7404 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7405 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7406 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7407 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7408 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7409 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7410 ;
7411 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7412 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7413 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7414 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7415 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7416 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7417 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7418 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7419 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7420 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7421 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7422 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7423 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7424 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7425 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7426 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7427 ; GFX940-TGSPLIT-NEXT: s_endpgm
7428 ;
7429 ; GFX11-WGP-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7430 ; GFX11-WGP: ; %bb.0: ; %entry
7431 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7432 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
7433 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7434 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7435 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7436 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7437 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
7438 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
7439 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7440 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
7441 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7442 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7443 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7444 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
7445 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7446 ; GFX11-WGP-NEXT: s_endpgm
7447 ;
7448 ; GFX11-CU-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7449 ; GFX11-CU: ; %bb.0: ; %entry
7450 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7451 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
7452 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7453 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7454 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7455 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7456 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
7457 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
7458 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7459 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
7460 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7461 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7462 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7463 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7464 ; GFX11-CU-NEXT: s_endpgm
7465 ;
7466 ; GFX12-WGP-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7467 ; GFX12-WGP: ; %bb.0: ; %entry
7468 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7469 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
7470 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7471 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7472 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7473 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7474 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
7475 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
7476 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7477 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
7478 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
7479 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7480 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7481 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
7482 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7483 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7484 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
7485 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7486 ; GFX12-WGP-NEXT: s_endpgm
7487 ;
7488 ; GFX12-CU-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
7489 ; GFX12-CU: ; %bb.0: ; %entry
7490 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7491 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
7492 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7493 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7494 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7495 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7496 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
7497 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
7498 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7499 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
7500 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7501 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
7502 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7503 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7504 ; GFX12-CU-NEXT: s_endpgm
7505 ptr addrspace(1) %out, i32 %in, i32 %old) {
7506 entry:
7507 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
7508 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
7509 %val0 = extractvalue { i32, i1 } %val, 0
7510 store i32 %val0, ptr addrspace(1) %out, align 4
7511 ret void
7512 }
7514 define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
7515 ; GFX6-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7516 ; GFX6: ; %bb.0: ; %entry
7517 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7518 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
7519 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
7520 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7521 ; GFX6-NEXT: s_mov_b32 s12, s5
7522 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
7523 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
7524 ; GFX6-NEXT: s_mov_b32 s11, -1
7525 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
7526 ; GFX6-NEXT: s_mov_b32 s5, s12
7527 ; GFX6-NEXT: s_mov_b32 s6, s11
7528 ; GFX6-NEXT: s_mov_b32 s7, s10
7529 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
7530 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
7531 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7532 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
7533 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7534 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
7535 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7536 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7537 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
7538 ; GFX6-NEXT: s_endpgm
7539 ;
7540 ; GFX7-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7541 ; GFX7: ; %bb.0: ; %entry
7542 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7543 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7544 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7545 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7546 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7547 ; GFX7-NEXT: s_mov_b32 s6, s4
7548 ; GFX7-NEXT: s_mov_b32 s7, s5
7549 ; GFX7-NEXT: s_mov_b32 s11, s12
7550 ; GFX7-NEXT: s_mov_b32 s10, s13
7551 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7552 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7553 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7554 ; GFX7-NEXT: s_mov_b32 s7, s10
7555 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7556 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7557 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7558 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7559 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7560 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7561 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7562 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7563 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7564 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7565 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7566 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7567 ; GFX7-NEXT: s_endpgm
7568 ;
7569 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7570 ; GFX10-WGP: ; %bb.0: ; %entry
7571 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
7572 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
7573 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7574 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
7575 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
7576 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7577 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7578 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
7579 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7580 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
7581 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7582 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7583 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7584 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
7585 ; GFX10-WGP-NEXT: buffer_gl0_inv
7586 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
7587 ; GFX10-WGP-NEXT: s_endpgm
7588 ;
7589 ; GFX10-CU-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7590 ; GFX10-CU: ; %bb.0: ; %entry
7591 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
7592 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
7593 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7594 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
7595 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
7596 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7597 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7598 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
7599 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7600 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
7601 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7602 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7603 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7604 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
7605 ; GFX10-CU-NEXT: s_endpgm
7606 ;
7607 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7608 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7609 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7610 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7611 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7612 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7613 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
7614 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
7615 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
7616 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
7617 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
7618 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
7619 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
7620 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7621 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
7622 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
7623 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7624 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
7625 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7626 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7627 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7628 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7629 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
7630 ; SKIP-CACHE-INV-NEXT: s_endpgm
7631 ;
7632 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7633 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7634 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7635 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7636 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7637 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7638 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7639 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7640 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7641 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7642 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7643 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7644 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7645 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7646 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7647 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7648 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7649 ;
7650 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7651 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7652 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7653 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7654 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7655 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7656 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7657 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7658 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7659 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7660 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7661 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7662 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7663 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7664 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7665 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7666 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7667 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7668 ;
7669 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7670 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7671 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7672 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7673 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7674 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7675 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7676 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7677 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7678 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7679 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7680 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7681 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7682 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7683 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7684 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7685 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7686 ;
7687 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7688 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7689 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7690 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7691 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7692 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7693 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7694 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7695 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7696 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7697 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7698 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7699 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7700 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7701 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7702 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7703 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7704 ; GFX940-TGSPLIT-NEXT: s_endpgm
7705 ;
7706 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7707 ; GFX11-WGP: ; %bb.0: ; %entry
7708 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7709 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
7710 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7711 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7712 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7713 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7714 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
7715 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
7716 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7717 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
7718 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7719 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7720 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7721 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
7722 ; GFX11-WGP-NEXT: buffer_gl0_inv
7723 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7724 ; GFX11-WGP-NEXT: s_endpgm
7725 ;
7726 ; GFX11-CU-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7727 ; GFX11-CU: ; %bb.0: ; %entry
7728 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7729 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
7730 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7731 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7732 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7733 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7734 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
7735 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
7736 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7737 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
7738 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
7739 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
7740 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
7741 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7742 ; GFX11-CU-NEXT: s_endpgm
7743 ;
7744 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7745 ; GFX12-WGP: ; %bb.0: ; %entry
7746 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7747 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
7748 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7749 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7750 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7751 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
7752 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
7753 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
7754 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7755 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
7756 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
7757 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7758 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7759 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
7760 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
7761 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
7762 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
7763 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
7764 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
7765 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
7766 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
7767 ; GFX12-WGP-NEXT: s_endpgm
7768 ;
7769 ; GFX12-CU-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
7770 ; GFX12-CU: ; %bb.0: ; %entry
7771 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
7772 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
7773 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7774 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
7775 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
7776 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
7777 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
7778 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
7779 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7780 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
7781 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
7782 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
7783 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
7784 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
7785 ; GFX12-CU-NEXT: s_endpgm
7786 ptr addrspace(1) %out, i32 %in, i32 %old) {
7787 entry:
7788 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
7789 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
7790 %val0 = extractvalue { i32, i1 } %val, 0
7791 store i32 %val0, ptr addrspace(1) %out, align 4
7792 ret void
7793 }
7795 define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
7796 ; GFX6-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7797 ; GFX6: ; %bb.0: ; %entry
7798 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7799 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
7800 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
7801 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7802 ; GFX6-NEXT: s_mov_b32 s12, s5
7803 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
7804 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
7805 ; GFX6-NEXT: s_mov_b32 s11, -1
7806 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
7807 ; GFX6-NEXT: s_mov_b32 s5, s12
7808 ; GFX6-NEXT: s_mov_b32 s6, s11
7809 ; GFX6-NEXT: s_mov_b32 s7, s10
7810 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
7811 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
7812 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7813 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
7814 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
7815 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
7816 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7817 ; GFX6-NEXT: s_waitcnt vmcnt(0)
7818 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
7819 ; GFX6-NEXT: s_endpgm
7820 ;
7821 ; GFX7-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7822 ; GFX7: ; %bb.0: ; %entry
7823 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
7824 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
7825 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
7826 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
7827 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7828 ; GFX7-NEXT: s_mov_b32 s6, s4
7829 ; GFX7-NEXT: s_mov_b32 s7, s5
7830 ; GFX7-NEXT: s_mov_b32 s11, s12
7831 ; GFX7-NEXT: s_mov_b32 s10, s13
7832 ; GFX7-NEXT: s_add_u32 s6, s6, s11
7833 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
7834 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
7835 ; GFX7-NEXT: s_mov_b32 s7, s10
7836 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
7837 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
7838 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7839 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
7840 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
7841 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
7842 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
7843 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7844 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
7845 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
7846 ; GFX7-NEXT: s_waitcnt vmcnt(0)
7847 ; GFX7-NEXT: flat_store_dword v[0:1], v2
7848 ; GFX7-NEXT: s_endpgm
7849 ;
7850 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7851 ; GFX10-WGP: ; %bb.0: ; %entry
7852 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
7853 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
7854 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7855 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
7856 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
7857 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
7858 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
7859 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
7860 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7861 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
7862 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7863 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
7864 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7865 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
7866 ; GFX10-WGP-NEXT: buffer_gl0_inv
7867 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
7868 ; GFX10-WGP-NEXT: s_endpgm
7869 ;
7870 ; GFX10-CU-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7871 ; GFX10-CU: ; %bb.0: ; %entry
7872 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
7873 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
7874 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7875 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
7876 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
7877 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7878 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
7879 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
7880 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7881 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
7882 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
7883 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
7884 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
7885 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
7886 ; GFX10-CU-NEXT: s_endpgm
7887 ;
7888 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7889 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
7890 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
7891 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
7892 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
7893 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7894 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
7895 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
7896 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
7897 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
7898 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
7899 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
7900 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
7901 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
7902 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
7903 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
7904 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
7905 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
7906 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
7907 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7908 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
7909 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
7910 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
7911 ; SKIP-CACHE-INV-NEXT: s_endpgm
7912 ;
7913 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7914 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
7915 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7916 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7917 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7918 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7919 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7920 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7921 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7922 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7923 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7924 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7925 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7926 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7927 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7928 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7929 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
7930 ;
7931 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7932 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
7933 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
7934 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7935 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
7936 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
7937 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
7938 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7939 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
7940 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
7941 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7942 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7943 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7944 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
7945 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7946 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
7947 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
7948 ; GFX90A-TGSPLIT-NEXT: s_endpgm
7949 ;
7950 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7951 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
7952 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7953 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7954 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7955 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7956 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7957 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7958 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7959 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7960 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7961 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7962 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7963 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7964 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
7965 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7966 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
7967 ;
7968 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7969 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
7970 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
7971 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
7972 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
7973 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
7974 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
7975 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
7976 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
7977 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
7978 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
7979 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
7980 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
7981 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
7982 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
7983 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
7984 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
7985 ; GFX940-TGSPLIT-NEXT: s_endpgm
7986 ;
7987 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
7988 ; GFX11-WGP: ; %bb.0: ; %entry
7989 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
7990 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
7991 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
7992 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
7993 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
7994 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
7995 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
7996 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
7997 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
7998 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
7999 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8000 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8001 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8002 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
8003 ; GFX11-WGP-NEXT: buffer_gl0_inv
8004 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8005 ; GFX11-WGP-NEXT: s_endpgm
8006 ;
8007 ; GFX11-CU-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
8008 ; GFX11-CU: ; %bb.0: ; %entry
8009 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8010 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
8011 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8012 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8013 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8014 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8015 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
8016 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
8017 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8018 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
8019 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8020 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8021 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8022 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8023 ; GFX11-CU-NEXT: s_endpgm
8024 ;
8025 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
8026 ; GFX12-WGP: ; %bb.0: ; %entry
8027 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8028 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
8029 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8030 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8031 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8032 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8033 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
8034 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
8035 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8036 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
8037 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8038 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8039 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8040 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8041 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8042 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8043 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8044 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8045 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
8046 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8047 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8048 ; GFX12-WGP-NEXT: s_endpgm
8049 ;
8050 ; GFX12-CU-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
8051 ; GFX12-CU: ; %bb.0: ; %entry
8052 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8053 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
8054 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8055 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8056 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8057 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8058 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
8059 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
8060 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8061 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
8062 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8063 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
8064 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8065 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8066 ; GFX12-CU-NEXT: s_endpgm
8067 ptr addrspace(1) %out, i32 %in, i32 %old) {
8068 entry:
8069 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
8070 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
8071 %val0 = extractvalue { i32, i1 } %val, 0
8072 store i32 %val0, ptr addrspace(1) %out, align 4
8073 ret void
8074 }
8076 define amdgpu_kernel void @global_workgroup_monotonic_acquire_ret_cmpxchg(
8077 ; GFX6-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8078 ; GFX6: ; %bb.0: ; %entry
8079 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8080 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
8081 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
8082 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8083 ; GFX6-NEXT: s_mov_b32 s12, s5
8084 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
8085 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
8086 ; GFX6-NEXT: s_mov_b32 s11, -1
8087 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
8088 ; GFX6-NEXT: s_mov_b32 s5, s12
8089 ; GFX6-NEXT: s_mov_b32 s6, s11
8090 ; GFX6-NEXT: s_mov_b32 s7, s10
8091 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
8092 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
8093 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8094 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
8095 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8096 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8097 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8098 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
8099 ; GFX6-NEXT: s_endpgm
8100 ;
8101 ; GFX7-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8102 ; GFX7: ; %bb.0: ; %entry
8103 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8104 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8105 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8106 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8107 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8108 ; GFX7-NEXT: s_mov_b32 s6, s4
8109 ; GFX7-NEXT: s_mov_b32 s7, s5
8110 ; GFX7-NEXT: s_mov_b32 s11, s12
8111 ; GFX7-NEXT: s_mov_b32 s10, s13
8112 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8113 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8114 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8115 ; GFX7-NEXT: s_mov_b32 s7, s10
8116 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8117 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8118 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8119 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8120 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8121 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8122 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8123 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8124 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8125 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8126 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8127 ; GFX7-NEXT: s_endpgm
8128 ;
8129 ; GFX10-WGP-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8130 ; GFX10-WGP: ; %bb.0: ; %entry
8131 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
8132 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
8133 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8134 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
8135 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
8136 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8137 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8138 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
8139 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8140 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
8141 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8142 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
8143 ; GFX10-WGP-NEXT: buffer_gl0_inv
8144 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
8145 ; GFX10-WGP-NEXT: s_endpgm
8146 ;
8147 ; GFX10-CU-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8148 ; GFX10-CU: ; %bb.0: ; %entry
8149 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
8150 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
8151 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8152 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
8153 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
8154 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8155 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8156 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
8157 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8158 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
8159 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8160 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8161 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
8162 ; GFX10-CU-NEXT: s_endpgm
8163 ;
8164 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8165 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8166 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8167 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8168 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8169 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8170 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
8171 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
8172 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
8173 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
8174 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
8175 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
8176 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
8177 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8178 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
8179 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
8180 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8181 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
8182 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8183 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8184 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8185 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
8186 ; SKIP-CACHE-INV-NEXT: s_endpgm
8187 ;
8188 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8189 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8190 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8191 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8192 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8193 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8194 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8195 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8196 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8197 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8198 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8199 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8200 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8201 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8202 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8203 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8204 ;
8205 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8206 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8207 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8208 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8209 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8210 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8211 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8212 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8213 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8214 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8215 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8216 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8217 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8218 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8219 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8220 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8221 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8222 ;
8223 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8224 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8225 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8226 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8227 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8228 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8229 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8230 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8231 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8232 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8233 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8234 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8235 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8236 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8237 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8238 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8239 ;
8240 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8241 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8242 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8243 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8244 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8245 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8246 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8247 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8248 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8249 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8250 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8251 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8252 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8253 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8254 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8255 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8256 ; GFX940-TGSPLIT-NEXT: s_endpgm
8257 ;
8258 ; GFX11-WGP-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8259 ; GFX11-WGP: ; %bb.0: ; %entry
8260 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8261 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
8262 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8263 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8264 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8265 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8266 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
8267 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
8268 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8269 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
8270 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8271 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
8272 ; GFX11-WGP-NEXT: buffer_gl0_inv
8273 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8274 ; GFX11-WGP-NEXT: s_endpgm
8275 ;
8276 ; GFX11-CU-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8277 ; GFX11-CU: ; %bb.0: ; %entry
8278 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8279 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
8280 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8281 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8282 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8283 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8284 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
8285 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
8286 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8287 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
8288 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8289 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8290 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8291 ; GFX11-CU-NEXT: s_endpgm
8292 ;
8293 ; GFX12-WGP-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8294 ; GFX12-WGP: ; %bb.0: ; %entry
8295 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8296 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
8297 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8298 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8299 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8300 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8301 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
8302 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
8303 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8304 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
8305 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8306 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8307 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8308 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
8309 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8310 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8311 ; GFX12-WGP-NEXT: s_endpgm
8312 ;
8313 ; GFX12-CU-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
8314 ; GFX12-CU: ; %bb.0: ; %entry
8315 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8316 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
8317 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8318 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8319 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8320 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8321 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
8322 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
8323 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8324 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
8325 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
8326 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8327 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8328 ; GFX12-CU-NEXT: s_endpgm
8329 ptr addrspace(1) %out, i32 %in, i32 %old) {
8330 entry:
8331 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
8332 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire
8333 %val0 = extractvalue { i32, i1 } %val, 0
8334 store i32 %val0, ptr addrspace(1) %out, align 4
8335 ret void
8336 }
8338 define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
8339 ; GFX6-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8340 ; GFX6: ; %bb.0: ; %entry
8341 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8342 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
8343 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
8344 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8345 ; GFX6-NEXT: s_mov_b32 s12, s5
8346 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
8347 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
8348 ; GFX6-NEXT: s_mov_b32 s11, -1
8349 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
8350 ; GFX6-NEXT: s_mov_b32 s5, s12
8351 ; GFX6-NEXT: s_mov_b32 s6, s11
8352 ; GFX6-NEXT: s_mov_b32 s7, s10
8353 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
8354 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
8355 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8356 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
8357 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8358 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8359 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8360 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
8361 ; GFX6-NEXT: s_endpgm
8362 ;
8363 ; GFX7-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8364 ; GFX7: ; %bb.0: ; %entry
8365 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8366 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8367 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8368 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8369 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8370 ; GFX7-NEXT: s_mov_b32 s6, s4
8371 ; GFX7-NEXT: s_mov_b32 s7, s5
8372 ; GFX7-NEXT: s_mov_b32 s11, s12
8373 ; GFX7-NEXT: s_mov_b32 s10, s13
8374 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8375 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8376 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8377 ; GFX7-NEXT: s_mov_b32 s7, s10
8378 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8379 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8380 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8381 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8382 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8383 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8384 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8385 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8386 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8387 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8388 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8389 ; GFX7-NEXT: s_endpgm
8390 ;
8391 ; GFX10-WGP-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8392 ; GFX10-WGP: ; %bb.0: ; %entry
8393 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
8394 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
8395 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8396 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
8397 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
8398 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8399 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8400 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
8401 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8402 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
8403 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8404 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
8405 ; GFX10-WGP-NEXT: buffer_gl0_inv
8406 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
8407 ; GFX10-WGP-NEXT: s_endpgm
8408 ;
8409 ; GFX10-CU-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8410 ; GFX10-CU: ; %bb.0: ; %entry
8411 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
8412 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
8413 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8414 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
8415 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
8416 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8417 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8418 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
8419 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8420 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
8421 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8422 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8423 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
8424 ; GFX10-CU-NEXT: s_endpgm
8425 ;
8426 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8427 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8428 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8429 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8430 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8431 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8432 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
8433 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
8434 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
8435 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
8436 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
8437 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
8438 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
8439 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8440 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
8441 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
8442 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8443 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
8444 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8445 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8446 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8447 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
8448 ; SKIP-CACHE-INV-NEXT: s_endpgm
8449 ;
8450 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8451 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8452 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8453 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8454 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8455 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8456 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8457 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8458 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8459 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8460 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8461 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8462 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8463 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8464 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8465 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8466 ;
8467 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8468 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8469 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8470 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8471 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8472 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8473 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8474 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8475 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8476 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8477 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8478 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8479 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8480 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8481 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8482 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8483 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8484 ;
8485 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8486 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8487 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8488 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8489 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8490 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8491 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8492 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8493 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8494 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8495 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8496 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8497 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8498 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8499 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8500 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8501 ;
8502 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8503 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8504 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8505 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8506 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8507 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8508 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8509 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8510 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8511 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8512 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8513 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8514 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8515 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8516 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8517 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8518 ; GFX940-TGSPLIT-NEXT: s_endpgm
8519 ;
8520 ; GFX11-WGP-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8521 ; GFX11-WGP: ; %bb.0: ; %entry
8522 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8523 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
8524 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8525 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8526 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8527 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8528 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
8529 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
8530 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8531 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
8532 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8533 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
8534 ; GFX11-WGP-NEXT: buffer_gl0_inv
8535 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8536 ; GFX11-WGP-NEXT: s_endpgm
8537 ;
8538 ; GFX11-CU-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8539 ; GFX11-CU: ; %bb.0: ; %entry
8540 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8541 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
8542 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8543 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8544 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8545 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8546 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
8547 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
8548 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8549 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
8550 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8551 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8552 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8553 ; GFX11-CU-NEXT: s_endpgm
8554 ;
8555 ; GFX12-WGP-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8556 ; GFX12-WGP: ; %bb.0: ; %entry
8557 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8558 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
8559 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8560 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8561 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8562 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8563 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
8564 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
8565 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8566 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
8567 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8568 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8569 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8570 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
8571 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8572 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8573 ; GFX12-WGP-NEXT: s_endpgm
8574 ;
8575 ; GFX12-CU-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
8576 ; GFX12-CU: ; %bb.0: ; %entry
8577 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8578 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
8579 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8580 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8581 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8582 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8583 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
8584 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
8585 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8586 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
8587 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
8588 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8589 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8590 ; GFX12-CU-NEXT: s_endpgm
8591 ptr addrspace(1) %out, i32 %in, i32 %old) {
8592 entry:
8593 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
8594 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
8595 %val0 = extractvalue { i32, i1 } %val, 0
8596 store i32 %val0, ptr addrspace(1) %out, align 4
8597 ret void
8598 }
8600 define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
8601 ; GFX6-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8602 ; GFX6: ; %bb.0: ; %entry
8603 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8604 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
8605 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
8606 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8607 ; GFX6-NEXT: s_mov_b32 s12, s5
8608 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
8609 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
8610 ; GFX6-NEXT: s_mov_b32 s11, -1
8611 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
8612 ; GFX6-NEXT: s_mov_b32 s5, s12
8613 ; GFX6-NEXT: s_mov_b32 s6, s11
8614 ; GFX6-NEXT: s_mov_b32 s7, s10
8615 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
8616 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
8617 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8618 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
8619 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8620 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8621 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8622 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8623 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
8624 ; GFX6-NEXT: s_endpgm
8625 ;
8626 ; GFX7-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8627 ; GFX7: ; %bb.0: ; %entry
8628 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8629 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8630 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8631 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8632 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8633 ; GFX7-NEXT: s_mov_b32 s6, s4
8634 ; GFX7-NEXT: s_mov_b32 s7, s5
8635 ; GFX7-NEXT: s_mov_b32 s11, s12
8636 ; GFX7-NEXT: s_mov_b32 s10, s13
8637 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8638 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8639 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8640 ; GFX7-NEXT: s_mov_b32 s7, s10
8641 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8642 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8643 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8644 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8645 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8646 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8647 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8648 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8649 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8650 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8651 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8652 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8653 ; GFX7-NEXT: s_endpgm
8654 ;
8655 ; GFX10-WGP-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8656 ; GFX10-WGP: ; %bb.0: ; %entry
8657 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
8658 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
8659 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8660 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
8661 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
8662 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8663 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8664 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
8665 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8666 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
8667 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8668 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8669 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8670 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
8671 ; GFX10-WGP-NEXT: buffer_gl0_inv
8672 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
8673 ; GFX10-WGP-NEXT: s_endpgm
8674 ;
8675 ; GFX10-CU-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8676 ; GFX10-CU: ; %bb.0: ; %entry
8677 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
8678 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
8679 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8680 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
8681 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
8682 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8683 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8684 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
8685 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8686 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
8687 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8688 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8689 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8690 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
8691 ; GFX10-CU-NEXT: s_endpgm
8692 ;
8693 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8694 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8695 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8696 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8697 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8698 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8699 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
8700 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
8701 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
8702 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
8703 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
8704 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
8705 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
8706 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8707 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
8708 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
8709 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8710 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
8711 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8712 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8713 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8714 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8715 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
8716 ; SKIP-CACHE-INV-NEXT: s_endpgm
8717 ;
8718 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8719 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
8720 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8721 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8722 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8723 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8724 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8725 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8726 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8727 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8728 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8729 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8730 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8731 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8732 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8733 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8734 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
8735 ;
8736 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8737 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
8738 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
8739 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8740 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8741 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
8742 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
8743 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8744 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
8745 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
8746 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8747 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8748 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8749 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
8750 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8751 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
8752 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
8753 ; GFX90A-TGSPLIT-NEXT: s_endpgm
8754 ;
8755 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8756 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
8757 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8758 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8759 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8760 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8761 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8762 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8763 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8764 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8765 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8766 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8767 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8768 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8769 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
8770 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8771 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
8772 ;
8773 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8774 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
8775 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
8776 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
8777 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
8778 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
8779 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
8780 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
8781 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
8782 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
8783 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8784 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
8785 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8786 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
8787 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
8788 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
8789 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
8790 ; GFX940-TGSPLIT-NEXT: s_endpgm
8791 ;
8792 ; GFX11-WGP-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8793 ; GFX11-WGP: ; %bb.0: ; %entry
8794 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8795 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
8796 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8797 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8798 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8799 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
8800 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
8801 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
8802 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8803 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
8804 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8805 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8806 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8807 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
8808 ; GFX11-WGP-NEXT: buffer_gl0_inv
8809 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8810 ; GFX11-WGP-NEXT: s_endpgm
8811 ;
8812 ; GFX11-CU-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8813 ; GFX11-CU: ; %bb.0: ; %entry
8814 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8815 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
8816 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8817 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8818 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8819 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8820 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
8821 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
8822 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8823 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
8824 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
8825 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
8826 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
8827 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8828 ; GFX11-CU-NEXT: s_endpgm
8829 ;
8830 ; GFX12-WGP-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8831 ; GFX12-WGP: ; %bb.0: ; %entry
8832 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
8833 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
8834 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8835 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
8836 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
8837 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
8838 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
8839 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
8840 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8841 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
8842 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
8843 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8844 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8845 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
8846 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
8847 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
8848 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
8849 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
8850 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
8851 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
8852 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
8853 ; GFX12-WGP-NEXT: s_endpgm
8854 ;
8855 ; GFX12-CU-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
8856 ; GFX12-CU: ; %bb.0: ; %entry
8857 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
8858 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
8859 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
8860 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
8861 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
8862 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
8863 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
8864 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
8865 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8866 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
8867 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
8868 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
8869 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
8870 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
8871 ; GFX12-CU-NEXT: s_endpgm
8872 ptr addrspace(1) %out, i32 %in, i32 %old) {
8873 entry:
8874 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
8875 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
8876 %val0 = extractvalue { i32, i1 } %val, 0
8877 store i32 %val0, ptr addrspace(1) %out, align 4
8878 ret void
8879 }
8881 define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
8882 ; GFX6-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
8883 ; GFX6: ; %bb.0: ; %entry
8884 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8885 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
8886 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
8887 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8888 ; GFX6-NEXT: s_mov_b32 s12, s5
8889 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
8890 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
8891 ; GFX6-NEXT: s_mov_b32 s11, -1
8892 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
8893 ; GFX6-NEXT: s_mov_b32 s5, s12
8894 ; GFX6-NEXT: s_mov_b32 s6, s11
8895 ; GFX6-NEXT: s_mov_b32 s7, s10
8896 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
8897 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
8898 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8899 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
8900 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
8901 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8902 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8903 ; GFX6-NEXT: s_waitcnt vmcnt(0)
8904 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
8905 ; GFX6-NEXT: s_endpgm
8906 ;
8907 ; GFX7-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
8908 ; GFX7: ; %bb.0: ; %entry
8909 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
8910 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
8911 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
8912 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
8913 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8914 ; GFX7-NEXT: s_mov_b32 s6, s4
8915 ; GFX7-NEXT: s_mov_b32 s7, s5
8916 ; GFX7-NEXT: s_mov_b32 s11, s12
8917 ; GFX7-NEXT: s_mov_b32 s10, s13
8918 ; GFX7-NEXT: s_add_u32 s6, s6, s11
8919 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
8920 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
8921 ; GFX7-NEXT: s_mov_b32 s7, s10
8922 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
8923 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
8924 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
8925 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
8926 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
8927 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
8928 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
8929 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8930 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
8931 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
8932 ; GFX7-NEXT: s_waitcnt vmcnt(0)
8933 ; GFX7-NEXT: flat_store_dword v[0:1], v2
8934 ; GFX7-NEXT: s_endpgm
8935 ;
8936 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
8937 ; GFX10-WGP: ; %bb.0: ; %entry
8938 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
8939 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
8940 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8941 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
8942 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
8943 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
8944 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
8945 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
8946 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8947 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
8948 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
8949 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
8950 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8951 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
8952 ; GFX10-WGP-NEXT: buffer_gl0_inv
8953 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
8954 ; GFX10-WGP-NEXT: s_endpgm
8955 ;
8956 ; GFX10-CU-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
8957 ; GFX10-CU: ; %bb.0: ; %entry
8958 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
8959 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
8960 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
8961 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
8962 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
8963 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8964 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
8965 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
8966 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
8967 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
8968 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
8969 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
8970 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
8971 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
8972 ; GFX10-CU-NEXT: s_endpgm
8973 ;
8974 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
8975 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
8976 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
8977 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
8978 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
8979 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8980 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
8981 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
8982 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
8983 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
8984 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
8985 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
8986 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
8987 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
8988 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
8989 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
8990 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
8991 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
8992 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
8993 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8994 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
8995 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
8996 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
8997 ; SKIP-CACHE-INV-NEXT: s_endpgm
8998 ;
8999 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9000 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9001 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9002 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9003 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9004 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9005 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9006 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9007 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9008 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9009 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9010 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9011 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9012 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9013 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9014 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9015 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9016 ;
9017 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9018 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9019 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9020 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9021 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9022 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9023 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9024 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9025 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9026 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9027 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9028 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9029 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9030 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9031 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9032 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9033 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9034 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9035 ;
9036 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9037 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9038 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9039 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9040 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9041 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9042 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9043 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9044 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9045 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9046 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9047 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9048 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9049 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9050 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9051 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9052 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9053 ;
9054 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9055 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9056 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9057 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9058 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9059 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9060 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9061 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9062 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9063 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9064 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9065 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9066 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9067 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9068 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9069 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9070 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9071 ; GFX940-TGSPLIT-NEXT: s_endpgm
9072 ;
9073 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9074 ; GFX11-WGP: ; %bb.0: ; %entry
9075 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9076 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
9077 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9078 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9079 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9080 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9081 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
9082 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
9083 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9084 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
9085 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9086 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9087 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9088 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
9089 ; GFX11-WGP-NEXT: buffer_gl0_inv
9090 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9091 ; GFX11-WGP-NEXT: s_endpgm
9092 ;
9093 ; GFX11-CU-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9094 ; GFX11-CU: ; %bb.0: ; %entry
9095 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9096 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
9097 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9098 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9099 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9100 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9101 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
9102 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
9103 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9104 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
9105 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9106 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9107 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9108 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9109 ; GFX11-CU-NEXT: s_endpgm
9110 ;
9111 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9112 ; GFX12-WGP: ; %bb.0: ; %entry
9113 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9114 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
9115 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9116 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9117 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9118 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9119 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
9120 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
9121 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9122 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
9123 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9124 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9125 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9126 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9127 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9128 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9129 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9130 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9131 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
9132 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9133 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9134 ; GFX12-WGP-NEXT: s_endpgm
9135 ;
9136 ; GFX12-CU-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
9137 ; GFX12-CU: ; %bb.0: ; %entry
9138 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9139 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
9140 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9141 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9142 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9143 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9144 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
9145 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
9146 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9147 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
9148 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9149 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
9150 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9151 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9152 ; GFX12-CU-NEXT: s_endpgm
9153 ptr addrspace(1) %out, i32 %in, i32 %old) {
9154 entry:
9155 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
9156 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
9157 %val0 = extractvalue { i32, i1 } %val, 0
9158 store i32 %val0, ptr addrspace(1) %out, align 4
9159 ret void
9160 }
9162 define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
9163 ; GFX6-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9164 ; GFX6: ; %bb.0: ; %entry
9165 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9166 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
9167 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
9168 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9169 ; GFX6-NEXT: s_mov_b32 s12, s5
9170 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
9171 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
9172 ; GFX6-NEXT: s_mov_b32 s11, -1
9173 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
9174 ; GFX6-NEXT: s_mov_b32 s5, s12
9175 ; GFX6-NEXT: s_mov_b32 s6, s11
9176 ; GFX6-NEXT: s_mov_b32 s7, s10
9177 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
9178 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
9179 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9180 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
9181 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9182 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9183 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9184 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9185 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
9186 ; GFX6-NEXT: s_endpgm
9187 ;
9188 ; GFX7-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9189 ; GFX7: ; %bb.0: ; %entry
9190 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9191 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9192 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9193 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9194 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9195 ; GFX7-NEXT: s_mov_b32 s6, s4
9196 ; GFX7-NEXT: s_mov_b32 s7, s5
9197 ; GFX7-NEXT: s_mov_b32 s11, s12
9198 ; GFX7-NEXT: s_mov_b32 s10, s13
9199 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9200 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9201 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9202 ; GFX7-NEXT: s_mov_b32 s7, s10
9203 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9204 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9205 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9206 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9207 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9208 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9209 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9210 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9211 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9212 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9213 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9214 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9215 ; GFX7-NEXT: s_endpgm
9216 ;
9217 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9218 ; GFX10-WGP: ; %bb.0: ; %entry
9219 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
9220 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
9221 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9222 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
9223 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
9224 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9225 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9226 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
9227 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9228 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
9229 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9230 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9231 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9232 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
9233 ; GFX10-WGP-NEXT: buffer_gl0_inv
9234 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
9235 ; GFX10-WGP-NEXT: s_endpgm
9236 ;
9237 ; GFX10-CU-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9238 ; GFX10-CU: ; %bb.0: ; %entry
9239 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
9240 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
9241 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9242 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
9243 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
9244 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9245 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9246 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
9247 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9248 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
9249 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9250 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9251 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9252 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
9253 ; GFX10-CU-NEXT: s_endpgm
9254 ;
9255 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9256 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9257 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9258 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9259 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9260 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9261 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
9262 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
9263 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
9264 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
9265 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
9266 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
9267 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
9268 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9269 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
9270 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
9271 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9272 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
9273 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9274 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9275 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9276 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9277 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
9278 ; SKIP-CACHE-INV-NEXT: s_endpgm
9279 ;
9280 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9281 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9282 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9283 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9284 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9285 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9286 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9287 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9288 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9289 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9290 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9291 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9292 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9293 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9294 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9295 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9296 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9297 ;
9298 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9299 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9300 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9301 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9302 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9303 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9304 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9305 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9306 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9307 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9308 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9309 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9310 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9311 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9312 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9313 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9314 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9315 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9316 ;
9317 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9318 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9319 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9320 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9321 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9322 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9323 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9324 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9325 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9326 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9327 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9328 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9329 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9330 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9331 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9332 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9333 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9334 ;
9335 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9336 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9337 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9338 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9339 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9340 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9341 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9342 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9343 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9344 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9345 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9346 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9347 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9348 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9349 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9350 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9351 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9352 ; GFX940-TGSPLIT-NEXT: s_endpgm
9353 ;
9354 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9355 ; GFX11-WGP: ; %bb.0: ; %entry
9356 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9357 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
9358 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9359 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9360 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9361 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9362 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
9363 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
9364 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9365 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
9366 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9367 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9368 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9369 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
9370 ; GFX11-WGP-NEXT: buffer_gl0_inv
9371 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9372 ; GFX11-WGP-NEXT: s_endpgm
9373 ;
9374 ; GFX11-CU-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9375 ; GFX11-CU: ; %bb.0: ; %entry
9376 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9377 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
9378 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9379 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9380 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9381 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9382 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
9383 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
9384 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9385 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
9386 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9387 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9388 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9389 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9390 ; GFX11-CU-NEXT: s_endpgm
9391 ;
9392 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9393 ; GFX12-WGP: ; %bb.0: ; %entry
9394 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9395 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
9396 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9397 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9398 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9399 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9400 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
9401 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
9402 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9403 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
9404 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9405 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9406 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9407 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9408 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9409 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9410 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9411 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9412 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
9413 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9414 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9415 ; GFX12-WGP-NEXT: s_endpgm
9416 ;
9417 ; GFX12-CU-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
9418 ; GFX12-CU: ; %bb.0: ; %entry
9419 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9420 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
9421 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9422 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9423 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9424 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9425 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
9426 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
9427 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9428 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
9429 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9430 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
9431 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9432 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9433 ; GFX12-CU-NEXT: s_endpgm
9434 ptr addrspace(1) %out, i32 %in, i32 %old) {
9435 entry:
9436 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
9437 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
9438 %val0 = extractvalue { i32, i1 } %val, 0
9439 store i32 %val0, ptr addrspace(1) %out, align 4
9440 ret void
9441 }
9443 define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_ret_cmpxchg(
9444 ; GFX6-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9445 ; GFX6: ; %bb.0: ; %entry
9446 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9447 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
9448 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
9449 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9450 ; GFX6-NEXT: s_mov_b32 s12, s5
9451 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
9452 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
9453 ; GFX6-NEXT: s_mov_b32 s11, -1
9454 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
9455 ; GFX6-NEXT: s_mov_b32 s5, s12
9456 ; GFX6-NEXT: s_mov_b32 s6, s11
9457 ; GFX6-NEXT: s_mov_b32 s7, s10
9458 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
9459 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
9460 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9461 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
9462 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9463 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9464 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9465 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9466 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
9467 ; GFX6-NEXT: s_endpgm
9468 ;
9469 ; GFX7-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9470 ; GFX7: ; %bb.0: ; %entry
9471 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9472 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9473 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9474 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9475 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9476 ; GFX7-NEXT: s_mov_b32 s6, s4
9477 ; GFX7-NEXT: s_mov_b32 s7, s5
9478 ; GFX7-NEXT: s_mov_b32 s11, s12
9479 ; GFX7-NEXT: s_mov_b32 s10, s13
9480 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9481 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9482 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9483 ; GFX7-NEXT: s_mov_b32 s7, s10
9484 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9485 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9486 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9487 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9488 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9489 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9490 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9491 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9492 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9493 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9494 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9495 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9496 ; GFX7-NEXT: s_endpgm
9497 ;
9498 ; GFX10-WGP-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9499 ; GFX10-WGP: ; %bb.0: ; %entry
9500 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
9501 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
9502 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9503 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
9504 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
9505 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9506 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9507 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
9508 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9509 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
9510 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9511 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9512 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9513 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
9514 ; GFX10-WGP-NEXT: buffer_gl0_inv
9515 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
9516 ; GFX10-WGP-NEXT: s_endpgm
9517 ;
9518 ; GFX10-CU-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9519 ; GFX10-CU: ; %bb.0: ; %entry
9520 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
9521 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
9522 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9523 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
9524 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
9525 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9526 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9527 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
9528 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9529 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
9530 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9531 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9532 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9533 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
9534 ; GFX10-CU-NEXT: s_endpgm
9535 ;
9536 ; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9537 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9538 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9539 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9540 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9541 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9542 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
9543 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
9544 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
9545 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
9546 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
9547 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
9548 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
9549 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9550 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
9551 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
9552 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9553 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
9554 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9555 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9556 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9557 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9558 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
9559 ; SKIP-CACHE-INV-NEXT: s_endpgm
9560 ;
9561 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9562 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9563 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9564 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9565 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9566 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9567 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9568 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9569 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9570 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9571 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9572 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9573 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9574 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9575 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9576 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9577 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9578 ;
9579 ; GFX90A-TGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9580 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9581 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9582 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9583 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9584 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9585 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9586 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9587 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9588 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9589 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9590 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9591 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9592 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9593 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9594 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9595 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9596 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9597 ;
9598 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9599 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9600 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9601 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9602 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9603 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9604 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9605 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9606 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9607 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9608 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9609 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9610 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9611 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9612 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9613 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9614 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9615 ;
9616 ; GFX940-TGSPLIT-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9617 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9618 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9619 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9620 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9621 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9622 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9623 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9624 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9625 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9626 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9627 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9628 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9629 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9630 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9631 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9632 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9633 ; GFX940-TGSPLIT-NEXT: s_endpgm
9634 ;
9635 ; GFX11-WGP-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9636 ; GFX11-WGP: ; %bb.0: ; %entry
9637 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9638 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
9639 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9640 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9641 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9642 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9643 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
9644 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
9645 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9646 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
9647 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9648 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9649 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9650 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
9651 ; GFX11-WGP-NEXT: buffer_gl0_inv
9652 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9653 ; GFX11-WGP-NEXT: s_endpgm
9654 ;
9655 ; GFX11-CU-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9656 ; GFX11-CU: ; %bb.0: ; %entry
9657 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9658 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
9659 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9660 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9661 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9662 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9663 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
9664 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
9665 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9666 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
9667 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9668 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9669 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9670 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9671 ; GFX11-CU-NEXT: s_endpgm
9672 ;
9673 ; GFX12-WGP-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9674 ; GFX12-WGP: ; %bb.0: ; %entry
9675 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9676 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
9677 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9678 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9679 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9680 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9681 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
9682 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
9683 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9684 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
9685 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9686 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9687 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9688 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9689 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9690 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9691 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9692 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9693 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
9694 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9695 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9696 ; GFX12-WGP-NEXT: s_endpgm
9697 ;
9698 ; GFX12-CU-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
9699 ; GFX12-CU: ; %bb.0: ; %entry
9700 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9701 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
9702 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9703 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9704 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9705 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9706 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
9707 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
9708 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9709 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
9710 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9711 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
9712 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9713 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9714 ; GFX12-CU-NEXT: s_endpgm
9715 ptr addrspace(1) %out, i32 %in, i32 %old) {
9716 entry:
9717 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
9718 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst
9719 %val0 = extractvalue { i32, i1 } %val, 0
9720 store i32 %val0, ptr addrspace(1) %out, align 4
9721 ret void
9722 }
9724 define amdgpu_kernel void @global_workgroup_acquire_seq_cst_ret_cmpxchg(
9725 ; GFX6-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9726 ; GFX6: ; %bb.0: ; %entry
9727 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9728 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
9729 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
9730 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9731 ; GFX6-NEXT: s_mov_b32 s12, s5
9732 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
9733 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
9734 ; GFX6-NEXT: s_mov_b32 s11, -1
9735 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
9736 ; GFX6-NEXT: s_mov_b32 s5, s12
9737 ; GFX6-NEXT: s_mov_b32 s6, s11
9738 ; GFX6-NEXT: s_mov_b32 s7, s10
9739 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
9740 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
9741 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9742 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
9743 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
9744 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9745 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9746 ; GFX6-NEXT: s_waitcnt vmcnt(0)
9747 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
9748 ; GFX6-NEXT: s_endpgm
9749 ;
9750 ; GFX7-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9751 ; GFX7: ; %bb.0: ; %entry
9752 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
9753 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
9754 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
9755 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
9756 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9757 ; GFX7-NEXT: s_mov_b32 s6, s4
9758 ; GFX7-NEXT: s_mov_b32 s7, s5
9759 ; GFX7-NEXT: s_mov_b32 s11, s12
9760 ; GFX7-NEXT: s_mov_b32 s10, s13
9761 ; GFX7-NEXT: s_add_u32 s6, s6, s11
9762 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
9763 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
9764 ; GFX7-NEXT: s_mov_b32 s7, s10
9765 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
9766 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
9767 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9768 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
9769 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
9770 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
9771 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
9772 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9773 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
9774 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
9775 ; GFX7-NEXT: s_waitcnt vmcnt(0)
9776 ; GFX7-NEXT: flat_store_dword v[0:1], v2
9777 ; GFX7-NEXT: s_endpgm
9778 ;
9779 ; GFX10-WGP-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9780 ; GFX10-WGP: ; %bb.0: ; %entry
9781 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
9782 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
9783 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9784 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
9785 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
9786 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
9787 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
9788 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
9789 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
9791 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9792 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9793 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9794 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
9795 ; GFX10-WGP-NEXT: buffer_gl0_inv
9796 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
9797 ; GFX10-WGP-NEXT: s_endpgm
9798 ;
9799 ; GFX10-CU-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9800 ; GFX10-CU: ; %bb.0: ; %entry
9801 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
9802 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
9803 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9804 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
9805 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
9806 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9807 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
9808 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
9809 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9810 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
9811 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
9812 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
9813 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
9814 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
9815 ; GFX10-CU-NEXT: s_endpgm
9816 ;
9817 ; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9818 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
9819 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
9820 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
9821 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
9822 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9823 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
9824 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
9825 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
9826 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
9827 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
9828 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
9829 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
9830 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
9831 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
9832 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
9833 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
9834 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
9835 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
9836 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9837 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
9838 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
9839 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
9840 ; SKIP-CACHE-INV-NEXT: s_endpgm
9841 ;
9842 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9843 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
9844 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9845 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9846 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9847 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9848 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9849 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9850 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9851 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9852 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9853 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9854 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9855 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9856 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9857 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9858 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
9859 ;
9860 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9861 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
9862 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
9863 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9864 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
9865 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
9866 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
9867 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9868 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
9869 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
9870 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9871 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9872 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9873 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
9874 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9875 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
9876 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
9877 ; GFX90A-TGSPLIT-NEXT: s_endpgm
9878 ;
9879 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9880 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
9881 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9882 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9883 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9884 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9885 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9886 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9887 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9888 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9889 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9890 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9891 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9892 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9893 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
9894 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9895 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
9896 ;
9897 ; GFX940-TGSPLIT-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9898 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
9899 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
9900 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
9901 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
9902 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
9903 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
9904 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
9905 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
9906 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
9907 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
9908 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
9909 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9910 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
9911 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
9912 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
9913 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
9914 ; GFX940-TGSPLIT-NEXT: s_endpgm
9915 ;
9916 ; GFX11-WGP-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9917 ; GFX11-WGP: ; %bb.0: ; %entry
9918 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9919 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
9920 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9921 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9922 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9923 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
9924 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
9925 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
9926 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9927 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
9928 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
9929 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
9930 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9931 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
9932 ; GFX11-WGP-NEXT: buffer_gl0_inv
9933 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9934 ; GFX11-WGP-NEXT: s_endpgm
9935 ;
9936 ; GFX11-CU-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9937 ; GFX11-CU: ; %bb.0: ; %entry
9938 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9939 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
9940 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9941 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9942 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9943 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9944 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
9945 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
9946 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9947 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
9948 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
9949 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
9950 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
9951 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9952 ; GFX11-CU-NEXT: s_endpgm
9953 ;
9954 ; GFX12-WGP-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9955 ; GFX12-WGP: ; %bb.0: ; %entry
9956 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
9957 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
9958 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9959 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
9960 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
9961 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
9962 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
9963 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
9964 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9965 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
9966 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
9967 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9968 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9969 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
9970 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
9971 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
9972 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
9973 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
9974 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
9975 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
9976 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
9977 ; GFX12-WGP-NEXT: s_endpgm
9978 ;
9979 ; GFX12-CU-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
9980 ; GFX12-CU: ; %bb.0: ; %entry
9981 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
9982 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
9983 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
9984 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
9985 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
9986 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
9987 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
9988 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
9989 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
9990 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
9991 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
9992 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
9993 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
9994 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
9995 ; GFX12-CU-NEXT: s_endpgm
9996 ptr addrspace(1) %out, i32 %in, i32 %old) {
9997 entry:
9998 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
9999 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst
10000 %val0 = extractvalue { i32, i1 } %val, 0
10001 store i32 %val0, ptr addrspace(1) %out, align 4
10002 ret void
10003 }
10005 define amdgpu_kernel void @global_workgroup_release_seq_cst_ret_cmpxchg(
10006 ; GFX6-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10007 ; GFX6: ; %bb.0: ; %entry
10008 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10009 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
10010 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
10011 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10012 ; GFX6-NEXT: s_mov_b32 s12, s5
10013 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
10014 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
10015 ; GFX6-NEXT: s_mov_b32 s11, -1
10016 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
10017 ; GFX6-NEXT: s_mov_b32 s5, s12
10018 ; GFX6-NEXT: s_mov_b32 s6, s11
10019 ; GFX6-NEXT: s_mov_b32 s7, s10
10020 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
10021 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
10022 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10023 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
10024 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10025 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
10026 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10027 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10028 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
10029 ; GFX6-NEXT: s_endpgm
10030 ;
10031 ; GFX7-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10032 ; GFX7: ; %bb.0: ; %entry
10033 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10034 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10035 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10036 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10037 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10038 ; GFX7-NEXT: s_mov_b32 s6, s4
10039 ; GFX7-NEXT: s_mov_b32 s7, s5
10040 ; GFX7-NEXT: s_mov_b32 s11, s12
10041 ; GFX7-NEXT: s_mov_b32 s10, s13
10042 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10043 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10044 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10045 ; GFX7-NEXT: s_mov_b32 s7, s10
10046 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10047 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10048 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10049 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10050 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10051 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10052 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10053 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10054 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10055 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10056 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10057 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10058 ; GFX7-NEXT: s_endpgm
10059 ;
10060 ; GFX10-WGP-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10061 ; GFX10-WGP: ; %bb.0: ; %entry
10062 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
10063 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
10064 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10065 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
10066 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
10067 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10068 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10069 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
10070 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10071 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
10072 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10073 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10074 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10075 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10076 ; GFX10-WGP-NEXT: buffer_gl0_inv
10077 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
10078 ; GFX10-WGP-NEXT: s_endpgm
10079 ;
10080 ; GFX10-CU-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10081 ; GFX10-CU: ; %bb.0: ; %entry
10082 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
10083 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
10084 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10085 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
10086 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
10087 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10088 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10089 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
10090 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10091 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
10092 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10093 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10094 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
10095 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
10096 ; GFX10-CU-NEXT: s_endpgm
10097 ;
10098 ; SKIP-CACHE-INV-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10099 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10100 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10101 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10102 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10103 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10104 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
10105 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
10106 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
10107 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
10108 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
10109 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
10110 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
10111 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10112 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
10113 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
10114 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10115 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
10116 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10117 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10118 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10119 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
10120 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
10121 ; SKIP-CACHE-INV-NEXT: s_endpgm
10122 ;
10123 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10124 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10125 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10126 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10127 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10128 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10129 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10130 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10131 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10132 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10133 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10134 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10135 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10136 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10137 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10138 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10139 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10140 ;
10141 ; GFX90A-TGSPLIT-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10142 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10143 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10144 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10145 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10146 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10147 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10148 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10149 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10150 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10151 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10152 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10153 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10154 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10155 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10156 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10157 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10158 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10159 ;
10160 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10161 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10162 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10164 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10165 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10166 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10167 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10168 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10169 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10170 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10171 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10172 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10173 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10174 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10175 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10176 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10177 ;
10178 ; GFX940-TGSPLIT-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10179 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10180 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10181 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10182 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10183 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10184 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10185 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10186 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10187 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10188 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10189 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10190 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10191 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10192 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10193 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10194 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10195 ; GFX940-TGSPLIT-NEXT: s_endpgm
10196 ;
10197 ; GFX11-WGP-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10198 ; GFX11-WGP: ; %bb.0: ; %entry
10199 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10200 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
10201 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10202 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10203 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10204 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10205 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10206 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
10207 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10208 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
10209 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10210 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10211 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10212 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10213 ; GFX11-WGP-NEXT: buffer_gl0_inv
10214 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10215 ; GFX11-WGP-NEXT: s_endpgm
10216 ;
10217 ; GFX11-CU-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10218 ; GFX11-CU: ; %bb.0: ; %entry
10219 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10220 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
10221 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10222 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10223 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10224 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10225 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10226 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
10227 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10228 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
10229 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10230 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10231 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
10232 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10233 ; GFX11-CU-NEXT: s_endpgm
10234 ;
10235 ; GFX12-WGP-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10236 ; GFX12-WGP: ; %bb.0: ; %entry
10237 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10238 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
10239 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10240 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10241 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10242 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10243 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10244 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
10245 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10246 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
10247 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
10248 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10249 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10250 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10251 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10252 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
10253 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10254 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10255 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10256 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10257 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10258 ; GFX12-WGP-NEXT: s_endpgm
10259 ;
10260 ; GFX12-CU-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
10261 ; GFX12-CU: ; %bb.0: ; %entry
10262 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10263 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
10264 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10265 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10266 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10267 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10268 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10269 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
10270 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10271 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
10272 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
10273 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
10274 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
10275 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10276 ; GFX12-CU-NEXT: s_endpgm
10277 ptr addrspace(1) %out, i32 %in, i32 %old) {
10278 entry:
10279 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
10280 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst
10281 %val0 = extractvalue { i32, i1 } %val, 0
10282 store i32 %val0, ptr addrspace(1) %out, align 4
10283 ret void
10284 }
10286 define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_ret_cmpxchg(
10287 ; GFX6-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10288 ; GFX6: ; %bb.0: ; %entry
10289 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10290 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
10291 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
10292 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10293 ; GFX6-NEXT: s_mov_b32 s12, s5
10294 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
10295 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
10296 ; GFX6-NEXT: s_mov_b32 s11, -1
10297 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
10298 ; GFX6-NEXT: s_mov_b32 s5, s12
10299 ; GFX6-NEXT: s_mov_b32 s6, s11
10300 ; GFX6-NEXT: s_mov_b32 s7, s10
10301 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
10302 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
10303 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10304 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
10305 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10306 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
10307 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10308 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10309 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
10310 ; GFX6-NEXT: s_endpgm
10311 ;
10312 ; GFX7-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10313 ; GFX7: ; %bb.0: ; %entry
10314 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10315 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10316 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10317 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10318 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10319 ; GFX7-NEXT: s_mov_b32 s6, s4
10320 ; GFX7-NEXT: s_mov_b32 s7, s5
10321 ; GFX7-NEXT: s_mov_b32 s11, s12
10322 ; GFX7-NEXT: s_mov_b32 s10, s13
10323 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10324 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10325 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10326 ; GFX7-NEXT: s_mov_b32 s7, s10
10327 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10328 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10329 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10330 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10331 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10332 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10333 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10334 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10335 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10336 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10337 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10338 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10339 ; GFX7-NEXT: s_endpgm
10340 ;
10341 ; GFX10-WGP-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10342 ; GFX10-WGP: ; %bb.0: ; %entry
10343 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
10344 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
10345 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10346 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
10347 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
10348 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10349 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10350 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
10351 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10352 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
10353 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10354 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10355 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10356 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10357 ; GFX10-WGP-NEXT: buffer_gl0_inv
10358 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
10359 ; GFX10-WGP-NEXT: s_endpgm
10360 ;
10361 ; GFX10-CU-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10362 ; GFX10-CU: ; %bb.0: ; %entry
10363 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
10364 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
10365 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10366 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
10367 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
10368 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10369 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10370 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
10371 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10372 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
10373 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10374 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10375 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
10376 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
10377 ; GFX10-CU-NEXT: s_endpgm
10378 ;
10379 ; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10380 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10381 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10382 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10383 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10384 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10385 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
10386 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
10387 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
10388 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
10389 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
10390 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
10391 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
10392 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10393 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
10394 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
10395 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10396 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
10397 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10398 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10399 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10400 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
10401 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
10402 ; SKIP-CACHE-INV-NEXT: s_endpgm
10403 ;
10404 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10405 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10406 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10407 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10408 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10409 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10410 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10411 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10412 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10413 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10414 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10415 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10416 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10417 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10418 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10419 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10420 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10421 ;
10422 ; GFX90A-TGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10423 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10424 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10425 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10426 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10427 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10428 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10429 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10430 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10431 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10432 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10433 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10434 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10435 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10436 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10437 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10438 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10439 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10440 ;
10441 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10442 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10443 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10444 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10445 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10446 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10447 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10448 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10449 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10450 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10451 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10452 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10453 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10454 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10455 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10456 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10457 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10458 ;
10459 ; GFX940-TGSPLIT-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10460 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10461 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10462 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10463 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10464 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10465 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10466 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10467 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10468 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10469 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10470 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10471 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10472 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10473 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10474 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10475 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10476 ; GFX940-TGSPLIT-NEXT: s_endpgm
10477 ;
10478 ; GFX11-WGP-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10479 ; GFX11-WGP: ; %bb.0: ; %entry
10480 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10481 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
10482 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10483 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10484 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10485 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10486 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10487 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
10488 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10489 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
10490 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10491 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10492 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10493 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10494 ; GFX11-WGP-NEXT: buffer_gl0_inv
10495 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10496 ; GFX11-WGP-NEXT: s_endpgm
10497 ;
10498 ; GFX11-CU-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10499 ; GFX11-CU: ; %bb.0: ; %entry
10500 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10501 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
10502 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10503 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10504 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10505 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10506 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10507 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
10508 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10509 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
10510 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10511 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10512 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
10513 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10514 ; GFX11-CU-NEXT: s_endpgm
10515 ;
10516 ; GFX12-WGP-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10517 ; GFX12-WGP: ; %bb.0: ; %entry
10518 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10519 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
10520 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10521 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10522 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10523 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10524 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10525 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
10526 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10527 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
10528 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
10529 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10530 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10531 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10532 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10533 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
10534 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10535 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10536 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10537 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10538 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10539 ; GFX12-WGP-NEXT: s_endpgm
10540 ;
10541 ; GFX12-CU-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
10542 ; GFX12-CU: ; %bb.0: ; %entry
10543 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10544 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
10545 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10546 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10547 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10548 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10549 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10550 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
10551 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10552 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
10553 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
10554 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
10555 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
10556 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10557 ; GFX12-CU-NEXT: s_endpgm
10558 ptr addrspace(1) %out, i32 %in, i32 %old) {
10559 entry:
10560 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
10561 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst
10562 %val0 = extractvalue { i32, i1 } %val, 0
10563 store i32 %val0, ptr addrspace(1) %out, align 4
10564 ret void
10565 }
10567 define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
10568 ; GFX6-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10569 ; GFX6: ; %bb.0: ; %entry
10570 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10571 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
10572 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
10573 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10574 ; GFX6-NEXT: s_mov_b32 s12, s5
10575 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
10576 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
10577 ; GFX6-NEXT: s_mov_b32 s11, -1
10578 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
10579 ; GFX6-NEXT: s_mov_b32 s5, s12
10580 ; GFX6-NEXT: s_mov_b32 s6, s11
10581 ; GFX6-NEXT: s_mov_b32 s7, s10
10582 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
10583 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
10584 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10585 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
10586 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10587 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
10588 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10589 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10590 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
10591 ; GFX6-NEXT: s_endpgm
10592 ;
10593 ; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10594 ; GFX7: ; %bb.0: ; %entry
10595 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
10596 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
10597 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
10598 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
10599 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10600 ; GFX7-NEXT: s_mov_b32 s6, s4
10601 ; GFX7-NEXT: s_mov_b32 s7, s5
10602 ; GFX7-NEXT: s_mov_b32 s11, s12
10603 ; GFX7-NEXT: s_mov_b32 s10, s13
10604 ; GFX7-NEXT: s_add_u32 s6, s6, s11
10605 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
10606 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
10607 ; GFX7-NEXT: s_mov_b32 s7, s10
10608 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
10609 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
10610 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10611 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
10612 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10613 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10614 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10615 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
10616 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10617 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10618 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10619 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10620 ; GFX7-NEXT: s_endpgm
10621 ;
10622 ; GFX10-WGP-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10623 ; GFX10-WGP: ; %bb.0: ; %entry
10624 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
10625 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
10626 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10627 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
10628 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
10629 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10630 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
10631 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
10632 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10633 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
10634 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10635 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10636 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10637 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10638 ; GFX10-WGP-NEXT: buffer_gl0_inv
10639 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
10640 ; GFX10-WGP-NEXT: s_endpgm
10641 ;
10642 ; GFX10-CU-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10643 ; GFX10-CU: ; %bb.0: ; %entry
10644 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
10645 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
10646 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10647 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
10648 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
10649 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10650 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
10651 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
10652 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10653 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
10654 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10655 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
10656 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
10657 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
10658 ; GFX10-CU-NEXT: s_endpgm
10659 ;
10660 ; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10661 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10662 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
10663 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
10664 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
10665 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10666 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
10667 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
10668 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
10669 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
10670 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
10671 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
10672 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
10673 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
10674 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
10675 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
10676 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
10677 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
10678 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10679 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
10680 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
10681 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
10682 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
10683 ; SKIP-CACHE-INV-NEXT: s_endpgm
10684 ;
10685 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10686 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10687 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10688 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10689 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10690 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10691 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10692 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10693 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10694 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10695 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10696 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10697 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10698 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10699 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10700 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10701 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10702 ;
10703 ; GFX90A-TGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10704 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10705 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
10706 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10707 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
10708 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
10709 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
10710 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10711 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
10712 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
10713 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10714 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10715 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10716 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
10717 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10718 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
10719 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10720 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10721 ;
10722 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10723 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10724 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10725 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10726 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10727 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10728 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10729 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10730 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10731 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10732 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10733 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10734 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10735 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10736 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10737 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10738 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10739 ;
10740 ; GFX940-TGSPLIT-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10741 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10742 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
10743 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10744 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
10745 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
10746 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
10747 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10748 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
10749 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
10750 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
10751 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
10752 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10753 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
10754 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10755 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
10756 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10757 ; GFX940-TGSPLIT-NEXT: s_endpgm
10758 ;
10759 ; GFX11-WGP-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10760 ; GFX11-WGP: ; %bb.0: ; %entry
10761 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10762 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
10763 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10764 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10765 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10766 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10767 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
10768 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
10769 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10770 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
10771 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
10772 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
10773 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10774 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10775 ; GFX11-WGP-NEXT: buffer_gl0_inv
10776 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10777 ; GFX11-WGP-NEXT: s_endpgm
10778 ;
10779 ; GFX11-CU-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10780 ; GFX11-CU: ; %bb.0: ; %entry
10781 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10782 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
10783 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10784 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10785 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10786 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10787 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
10788 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
10789 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10790 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
10791 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
10792 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
10793 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
10794 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10795 ; GFX11-CU-NEXT: s_endpgm
10796 ;
10797 ; GFX12-WGP-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10798 ; GFX12-WGP: ; %bb.0: ; %entry
10799 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
10800 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
10801 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10802 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
10803 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
10804 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
10805 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
10806 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
10807 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10808 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
10809 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
10810 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10811 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10812 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
10813 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
10814 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
10815 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
10816 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
10817 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
10818 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
10819 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
10820 ; GFX12-WGP-NEXT: s_endpgm
10821 ;
10822 ; GFX12-CU-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
10823 ; GFX12-CU: ; %bb.0: ; %entry
10824 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
10825 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
10826 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
10827 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
10828 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
10829 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
10830 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
10831 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
10832 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
10833 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
10834 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
10835 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
10836 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
10837 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
10838 ; GFX12-CU-NEXT: s_endpgm
10839 ptr addrspace(1) %out, i32 %in, i32 %old) {
10840 entry:
10841 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
10842 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
10843 %val0 = extractvalue { i32, i1 } %val, 0
10844 store i32 %val0, ptr addrspace(1) %out, align 4
10845 ret void
10846 }
10848 define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
10849 ; GFX6-LABEL: global_workgroup_one_as_unordered_load:
10850 ; GFX6: ; %bb.0: ; %entry
10851 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
10852 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
10853 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
10854 ; GFX6-NEXT: s_mov_b32 s6, s9
10855 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
10856 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
10857 ; GFX6-NEXT: s_mov_b32 s13, -1
10858 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
10859 ; GFX6-NEXT: s_mov_b32 s9, s6
10860 ; GFX6-NEXT: s_mov_b32 s10, s13
10861 ; GFX6-NEXT: s_mov_b32 s11, s12
10862 ; GFX6-NEXT: s_mov_b32 s14, s5
10863 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
10864 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
10865 ; GFX6-NEXT: s_mov_b32 s5, s14
10866 ; GFX6-NEXT: s_mov_b32 s6, s13
10867 ; GFX6-NEXT: s_mov_b32 s7, s12
10868 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
10869 ; GFX6-NEXT: s_waitcnt vmcnt(0)
10870 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
10871 ; GFX6-NEXT: s_endpgm
10872 ;
10873 ; GFX7-LABEL: global_workgroup_one_as_unordered_load:
10874 ; GFX7: ; %bb.0: ; %entry
10875 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
10876 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10877 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
10878 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
10879 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
10880 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
10881 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
10882 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
10883 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
10884 ; GFX7-NEXT: s_waitcnt vmcnt(0)
10885 ; GFX7-NEXT: flat_store_dword v[0:1], v2
10886 ; GFX7-NEXT: s_endpgm
10887 ;
10888 ; GFX10-WGP-LABEL: global_workgroup_one_as_unordered_load:
10889 ; GFX10-WGP: ; %bb.0: ; %entry
10890 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
10891 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
10892 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10893 ; GFX10-WGP-NEXT: s_nop 0
10894 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10895 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
10896 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7]
10897 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
10898 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
10899 ; GFX10-WGP-NEXT: s_endpgm
10900 ;
10901 ; GFX10-CU-LABEL: global_workgroup_one_as_unordered_load:
10902 ; GFX10-CU: ; %bb.0: ; %entry
10903 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
10904 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
10905 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10906 ; GFX10-CU-NEXT: s_nop 0
10907 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10908 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
10909 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
10910 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
10911 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
10912 ; GFX10-CU-NEXT: s_endpgm
10913 ;
10914 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_unordered_load:
10915 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
10916 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
10917 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
10918 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
10919 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
10920 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
10921 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
10922 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
10923 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
10924 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
10925 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
10926 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
10927 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
10928 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
10929 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
10930 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
10931 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
10932 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
10933 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
10934 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
10935 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
10936 ; SKIP-CACHE-INV-NEXT: s_endpgm
10937 ;
10938 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_unordered_load:
10939 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
10940 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10941 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10942 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10943 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
10944 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10945 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10946 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
10947 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10948 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10949 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
10950 ;
10951 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_unordered_load:
10952 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
10953 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
10954 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10955 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
10956 ; GFX90A-TGSPLIT-NEXT: s_nop 0
10957 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
10958 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10959 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
10960 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10961 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
10962 ; GFX90A-TGSPLIT-NEXT: s_endpgm
10963 ;
10964 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_unordered_load:
10965 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
10966 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10967 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10968 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10969 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
10970 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10971 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10972 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3]
10973 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
10974 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10975 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
10976 ;
10977 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_unordered_load:
10978 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
10979 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
10980 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
10981 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
10982 ; GFX940-TGSPLIT-NEXT: s_nop 0
10983 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
10984 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
10985 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3]
10986 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
10987 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
10988 ; GFX940-TGSPLIT-NEXT: s_endpgm
10989 ;
10990 ; GFX11-WGP-LABEL: global_workgroup_one_as_unordered_load:
10991 ; GFX11-WGP: ; %bb.0: ; %entry
10992 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
10993 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
10994 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
10995 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
10996 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
10997 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3]
10998 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
10999 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11000 ; GFX11-WGP-NEXT: s_endpgm
11001 ;
11002 ; GFX11-CU-LABEL: global_workgroup_one_as_unordered_load:
11003 ; GFX11-CU: ; %bb.0: ; %entry
11004 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11005 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11006 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11007 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11008 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11009 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11010 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
11011 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11012 ; GFX11-CU-NEXT: s_endpgm
11013 ;
11014 ; GFX12-WGP-LABEL: global_workgroup_one_as_unordered_load:
11015 ; GFX12-WGP: ; %bb.0: ; %entry
11016 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11017 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11018 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11019 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11020 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11021 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3]
11022 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11023 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11024 ; GFX12-WGP-NEXT: s_endpgm
11025 ;
11026 ; GFX12-CU-LABEL: global_workgroup_one_as_unordered_load:
11027 ; GFX12-CU: ; %bb.0: ; %entry
11028 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11029 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11030 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11031 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11032 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11033 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11034 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
11035 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11036 ; GFX12-CU-NEXT: s_endpgm
11037 ptr addrspace(1) %in, ptr addrspace(1) %out) {
11038 entry:
11039 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") unordered, align 4
11040 store i32 %val, ptr addrspace(1) %out
11041 ret void
11042 }
11044 define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
11045 ; GFX6-LABEL: global_workgroup_one_as_monotonic_load:
11046 ; GFX6: ; %bb.0: ; %entry
11047 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
11048 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11049 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11050 ; GFX6-NEXT: s_mov_b32 s6, s9
11051 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
11052 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
11053 ; GFX6-NEXT: s_mov_b32 s13, -1
11054 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
11055 ; GFX6-NEXT: s_mov_b32 s9, s6
11056 ; GFX6-NEXT: s_mov_b32 s10, s13
11057 ; GFX6-NEXT: s_mov_b32 s11, s12
11058 ; GFX6-NEXT: s_mov_b32 s14, s5
11059 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11060 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11061 ; GFX6-NEXT: s_mov_b32 s5, s14
11062 ; GFX6-NEXT: s_mov_b32 s6, s13
11063 ; GFX6-NEXT: s_mov_b32 s7, s12
11064 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
11065 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11066 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
11067 ; GFX6-NEXT: s_endpgm
11068 ;
11069 ; GFX7-LABEL: global_workgroup_one_as_monotonic_load:
11070 ; GFX7: ; %bb.0: ; %entry
11071 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11072 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11073 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
11074 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11075 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11076 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11077 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
11078 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11079 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11080 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11081 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11082 ; GFX7-NEXT: s_endpgm
11083 ;
11084 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_load:
11085 ; GFX10-WGP: ; %bb.0: ; %entry
11086 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11087 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
11088 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11089 ; GFX10-WGP-NEXT: s_nop 0
11090 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11091 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11092 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
11093 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11094 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
11095 ; GFX10-WGP-NEXT: s_endpgm
11096 ;
11097 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_load:
11098 ; GFX10-CU: ; %bb.0: ; %entry
11099 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11100 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
11101 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11102 ; GFX10-CU-NEXT: s_nop 0
11103 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11104 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11105 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
11106 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11107 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
11108 ; GFX10-CU-NEXT: s_endpgm
11109 ;
11110 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_load:
11111 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11112 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
11113 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
11114 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11115 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
11116 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11117 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
11118 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
11119 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11120 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
11121 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
11122 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
11123 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
11124 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
11125 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
11126 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
11127 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
11128 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
11129 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
11130 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11131 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
11132 ; SKIP-CACHE-INV-NEXT: s_endpgm
11133 ;
11134 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_load:
11135 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11136 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11137 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11138 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11139 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11140 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11141 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11142 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
11143 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11144 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11145 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11146 ;
11147 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_load:
11148 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11149 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11150 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11151 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11152 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11153 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11154 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11155 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
11156 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11157 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11158 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11159 ;
11160 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_load:
11161 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11162 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11164 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11165 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11166 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11167 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11168 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11169 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11170 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11171 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11172 ;
11173 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_load:
11174 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11175 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11176 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11177 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11178 ; GFX940-TGSPLIT-NEXT: s_nop 0
11179 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11180 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11181 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11182 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11183 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11184 ; GFX940-TGSPLIT-NEXT: s_endpgm
11185 ;
11186 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_load:
11187 ; GFX11-WGP: ; %bb.0: ; %entry
11188 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11189 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
11190 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11191 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11192 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11193 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
11194 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11195 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11196 ; GFX11-WGP-NEXT: s_endpgm
11197 ;
11198 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_load:
11199 ; GFX11-CU: ; %bb.0: ; %entry
11200 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11201 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11202 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11203 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11204 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11205 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11206 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
11207 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11208 ; GFX11-CU-NEXT: s_endpgm
11209 ;
11210 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_load:
11211 ; GFX12-WGP: ; %bb.0: ; %entry
11212 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11213 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11214 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11215 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11216 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11217 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
11218 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11219 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11220 ; GFX12-WGP-NEXT: s_endpgm
11221 ;
11222 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_load:
11223 ; GFX12-CU: ; %bb.0: ; %entry
11224 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11225 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11226 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11227 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11228 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11229 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11230 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
11231 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11232 ; GFX12-CU-NEXT: s_endpgm
11233 ptr addrspace(1) %in, ptr addrspace(1) %out) {
11234 entry:
11235 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") monotonic, align 4
11236 store i32 %val, ptr addrspace(1) %out
11237 ret void
11238 }
11240 define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
11241 ; GFX6-LABEL: global_workgroup_one_as_acquire_load:
11242 ; GFX6: ; %bb.0: ; %entry
11243 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
11244 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11245 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11246 ; GFX6-NEXT: s_mov_b32 s6, s9
11247 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
11248 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
11249 ; GFX6-NEXT: s_mov_b32 s13, -1
11250 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
11251 ; GFX6-NEXT: s_mov_b32 s9, s6
11252 ; GFX6-NEXT: s_mov_b32 s10, s13
11253 ; GFX6-NEXT: s_mov_b32 s11, s12
11254 ; GFX6-NEXT: s_mov_b32 s14, s5
11255 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11256 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11257 ; GFX6-NEXT: s_mov_b32 s5, s14
11258 ; GFX6-NEXT: s_mov_b32 s6, s13
11259 ; GFX6-NEXT: s_mov_b32 s7, s12
11260 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
11261 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11262 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
11263 ; GFX6-NEXT: s_endpgm
11264 ;
11265 ; GFX7-LABEL: global_workgroup_one_as_acquire_load:
11266 ; GFX7: ; %bb.0: ; %entry
11267 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11268 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11269 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
11270 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11271 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11272 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11273 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
11274 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11275 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11276 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11277 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11278 ; GFX7-NEXT: s_endpgm
11279 ;
11280 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_load:
11281 ; GFX10-WGP: ; %bb.0: ; %entry
11282 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11283 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
11284 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11285 ; GFX10-WGP-NEXT: s_nop 0
11286 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11287 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11288 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
11289 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11290 ; GFX10-WGP-NEXT: buffer_gl0_inv
11291 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
11292 ; GFX10-WGP-NEXT: s_endpgm
11293 ;
11294 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_load:
11295 ; GFX10-CU: ; %bb.0: ; %entry
11296 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11297 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
11298 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11299 ; GFX10-CU-NEXT: s_nop 0
11300 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11301 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11302 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
11303 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11304 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
11305 ; GFX10-CU-NEXT: s_endpgm
11306 ;
11307 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_load:
11308 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11309 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
11310 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
11311 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11312 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
11313 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11314 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
11315 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
11316 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11317 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
11318 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
11319 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
11320 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
11321 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
11322 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
11323 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
11324 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
11325 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
11326 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
11327 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11328 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
11329 ; SKIP-CACHE-INV-NEXT: s_endpgm
11330 ;
11331 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_load:
11332 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11333 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11334 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11335 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11336 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11337 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11338 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11339 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
11340 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11341 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11342 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11343 ;
11344 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_load:
11345 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11346 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11347 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11348 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11349 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11350 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11351 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11352 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
11353 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11354 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11355 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11356 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11357 ;
11358 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_load:
11359 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11360 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11361 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11362 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11363 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11364 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11365 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11366 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11367 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11368 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11369 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11370 ;
11371 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_load:
11372 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11373 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11374 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11375 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11376 ; GFX940-TGSPLIT-NEXT: s_nop 0
11377 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11378 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11379 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11380 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11381 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
11382 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11383 ; GFX940-TGSPLIT-NEXT: s_endpgm
11384 ;
11385 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_load:
11386 ; GFX11-WGP: ; %bb.0: ; %entry
11387 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11388 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
11389 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11390 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11391 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11392 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
11393 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11394 ; GFX11-WGP-NEXT: buffer_gl0_inv
11395 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11396 ; GFX11-WGP-NEXT: s_endpgm
11397 ;
11398 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_load:
11399 ; GFX11-CU: ; %bb.0: ; %entry
11400 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11401 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11402 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11403 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11404 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11405 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11406 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
11407 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11408 ; GFX11-CU-NEXT: s_endpgm
11409 ;
11410 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_load:
11411 ; GFX12-WGP: ; %bb.0: ; %entry
11412 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11413 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11414 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11415 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11416 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11417 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
11418 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11419 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11420 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11421 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
11422 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11423 ; GFX12-WGP-NEXT: s_endpgm
11424 ;
11425 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_load:
11426 ; GFX12-CU: ; %bb.0: ; %entry
11427 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11428 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11429 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11430 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11431 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11432 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11433 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
11434 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11435 ; GFX12-CU-NEXT: s_endpgm
11436 ptr addrspace(1) %in, ptr addrspace(1) %out) {
11437 entry:
11438 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") acquire, align 4
11439 store i32 %val, ptr addrspace(1) %out
11440 ret void
11441 }
11443 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
11444 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_load:
11445 ; GFX6: ; %bb.0: ; %entry
11446 ; GFX6-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
11447 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11448 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11449 ; GFX6-NEXT: s_mov_b32 s6, s9
11450 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 killed $sgpr8_sgpr9
11451 ; GFX6-NEXT: s_mov_b32 s12, 0x100f000
11452 ; GFX6-NEXT: s_mov_b32 s13, -1
11453 ; GFX6-NEXT: ; kill: def $sgpr8 killed $sgpr8 def $sgpr8_sgpr9_sgpr10_sgpr11
11454 ; GFX6-NEXT: s_mov_b32 s9, s6
11455 ; GFX6-NEXT: s_mov_b32 s10, s13
11456 ; GFX6-NEXT: s_mov_b32 s11, s12
11457 ; GFX6-NEXT: s_mov_b32 s14, s5
11458 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11459 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11460 ; GFX6-NEXT: s_mov_b32 s5, s14
11461 ; GFX6-NEXT: s_mov_b32 s6, s13
11462 ; GFX6-NEXT: s_mov_b32 s7, s12
11463 ; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0
11464 ; GFX6-NEXT: s_waitcnt vmcnt(0)
11465 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
11466 ; GFX6-NEXT: s_endpgm
11467 ;
11468 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_load:
11469 ; GFX7: ; %bb.0: ; %entry
11470 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
11471 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11472 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
11473 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11474 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11475 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11476 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
11477 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
11478 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
11479 ; GFX7-NEXT: s_waitcnt vmcnt(0)
11480 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11481 ; GFX7-NEXT: s_endpgm
11482 ;
11483 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_load:
11484 ; GFX10-WGP: ; %bb.0: ; %entry
11485 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11486 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
11487 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11488 ; GFX10-WGP-NEXT: s_nop 0
11489 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11490 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11491 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11492 ; GFX10-WGP-NEXT: global_load_dword v1, v0, s[6:7] glc
11493 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
11494 ; GFX10-WGP-NEXT: buffer_gl0_inv
11495 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
11496 ; GFX10-WGP-NEXT: s_endpgm
11497 ;
11498 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_load:
11499 ; GFX10-CU: ; %bb.0: ; %entry
11500 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11501 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
11502 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11503 ; GFX10-CU-NEXT: s_nop 0
11504 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11505 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11506 ; GFX10-CU-NEXT: global_load_dword v1, v0, s[6:7]
11507 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
11508 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
11509 ; GFX10-CU-NEXT: s_endpgm
11510 ;
11511 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_load:
11512 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11513 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
11514 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
11515 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11516 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s5
11517 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11518 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, 0xf000
11519 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s9, -1
11520 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11521 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, s2
11522 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, s9
11523 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s8
11524 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s10, s1
11525 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
11526 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
11527 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s10
11528 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s9
11529 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s8
11530 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, off, s[4:7], 0
11531 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
11532 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
11533 ; SKIP-CACHE-INV-NEXT: s_endpgm
11534 ;
11535 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_load:
11536 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11537 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11538 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11539 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11540 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11541 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11542 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11543 ; GFX90A-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[6:7]
11544 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11545 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11546 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11547 ;
11548 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_load:
11549 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11550 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11551 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11552 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
11553 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11554 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11555 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11556 ; GFX90A-TGSPLIT-NEXT: global_load_dword v1, v0, s[6:7] glc
11557 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11558 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
11559 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11560 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11561 ;
11562 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_load:
11563 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11564 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11565 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11566 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11567 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11568 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11569 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11570 ; GFX940-NOTTGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11571 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
11572 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11573 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11574 ;
11575 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_load:
11576 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11577 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11578 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11579 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
11580 ; GFX940-TGSPLIT-NEXT: s_nop 0
11581 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11582 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11583 ; GFX940-TGSPLIT-NEXT: global_load_dword v1, v0, s[2:3] sc0
11584 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
11585 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
11586 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11587 ; GFX940-TGSPLIT-NEXT: s_endpgm
11588 ;
11589 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_load:
11590 ; GFX11-WGP: ; %bb.0: ; %entry
11591 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11592 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
11593 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11594 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11595 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
11596 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
11597 ; GFX11-WGP-NEXT: global_load_b32 v1, v0, s[2:3] glc
11598 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
11599 ; GFX11-WGP-NEXT: buffer_gl0_inv
11600 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11601 ; GFX11-WGP-NEXT: s_endpgm
11602 ;
11603 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_load:
11604 ; GFX11-CU: ; %bb.0: ; %entry
11605 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11606 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11607 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11608 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11609 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11610 ; GFX11-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11611 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
11612 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11613 ; GFX11-CU-NEXT: s_endpgm
11614 ;
11615 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_load:
11616 ; GFX12-WGP: ; %bb.0: ; %entry
11617 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11618 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11619 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11620 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11621 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11622 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11623 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11624 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
11625 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11626 ; GFX12-WGP-NEXT: global_load_b32 v1, v0, s[2:3] scope:SCOPE_SE
11627 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
11628 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
11629 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
11630 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
11631 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11632 ; GFX12-WGP-NEXT: s_endpgm
11633 ;
11634 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_load:
11635 ; GFX12-CU: ; %bb.0: ; %entry
11636 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11637 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11638 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
11639 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11640 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11641 ; GFX12-CU-NEXT: global_load_b32 v1, v0, s[2:3]
11642 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
11643 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11644 ; GFX12-CU-NEXT: s_endpgm
11645 ptr addrspace(1) %in, ptr addrspace(1) %out) {
11646 entry:
11647 %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") seq_cst, align 4
11648 store i32 %val, ptr addrspace(1) %out
11649 ret void
11650 }
11652 define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
11653 ; GFX6-LABEL: global_workgroup_one_as_unordered_store:
11654 ; GFX6: ; %bb.0: ; %entry
11655 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
11656 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11657 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11658 ; GFX6-NEXT: s_mov_b32 s11, s5
11659 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11660 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
11661 ; GFX6-NEXT: s_mov_b32 s10, -1
11662 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11663 ; GFX6-NEXT: s_mov_b32 s5, s11
11664 ; GFX6-NEXT: s_mov_b32 s6, s10
11665 ; GFX6-NEXT: s_mov_b32 s7, s9
11666 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
11667 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
11668 ; GFX6-NEXT: s_endpgm
11669 ;
11670 ; GFX7-LABEL: global_workgroup_one_as_unordered_store:
11671 ; GFX7: ; %bb.0: ; %entry
11672 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
11673 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
11674 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11675 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11676 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11677 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11678 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11679 ; GFX7-NEXT: s_endpgm
11680 ;
11681 ; GFX10-WGP-LABEL: global_workgroup_one_as_unordered_store:
11682 ; GFX10-WGP: ; %bb.0: ; %entry
11683 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11684 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
11685 ; GFX10-WGP-NEXT: s_nop 0
11686 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11687 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
11688 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11689 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
11690 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
11691 ; GFX10-WGP-NEXT: s_endpgm
11692 ;
11693 ; GFX10-CU-LABEL: global_workgroup_one_as_unordered_store:
11694 ; GFX10-CU: ; %bb.0: ; %entry
11695 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11696 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
11697 ; GFX10-CU-NEXT: s_nop 0
11698 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11699 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
11700 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11701 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
11702 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
11703 ; GFX10-CU-NEXT: s_endpgm
11704 ;
11705 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_unordered_store:
11706 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11707 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
11708 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
11709 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11710 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
11711 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
11712 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
11713 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
11714 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
11715 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
11716 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
11717 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
11718 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11719 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
11720 ; SKIP-CACHE-INV-NEXT: s_endpgm
11721 ;
11722 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_unordered_store:
11723 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11724 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11725 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
11726 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11727 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11728 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11729 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11730 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
11731 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11732 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11733 ;
11734 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_unordered_store:
11735 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11736 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11737 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
11738 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11739 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11740 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11741 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11742 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
11743 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11744 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11745 ;
11746 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_unordered_store:
11747 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11748 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11749 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
11750 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11751 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11752 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11753 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11754 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
11755 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11756 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11757 ;
11758 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_unordered_store:
11759 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11760 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11761 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
11762 ; GFX940-TGSPLIT-NEXT: s_nop 0
11763 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11764 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11765 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11766 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
11767 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11768 ; GFX940-TGSPLIT-NEXT: s_endpgm
11769 ;
11770 ; GFX11-WGP-LABEL: global_workgroup_one_as_unordered_store:
11771 ; GFX11-WGP: ; %bb.0: ; %entry
11772 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11773 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
11774 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11775 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
11776 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11777 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
11778 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11779 ; GFX11-WGP-NEXT: s_endpgm
11780 ;
11781 ; GFX11-CU-LABEL: global_workgroup_one_as_unordered_store:
11782 ; GFX11-CU: ; %bb.0: ; %entry
11783 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11784 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
11785 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11786 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11787 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11788 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
11789 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11790 ; GFX11-CU-NEXT: s_endpgm
11791 ;
11792 ; GFX12-WGP-LABEL: global_workgroup_one_as_unordered_store:
11793 ; GFX12-WGP: ; %bb.0: ; %entry
11794 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11795 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
11796 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11797 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11798 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11799 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
11800 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11801 ; GFX12-WGP-NEXT: s_endpgm
11802 ;
11803 ; GFX12-CU-LABEL: global_workgroup_one_as_unordered_store:
11804 ; GFX12-CU: ; %bb.0: ; %entry
11805 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11806 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
11807 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11808 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11809 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11810 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
11811 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11812 ; GFX12-CU-NEXT: s_endpgm
11813 i32 %in, ptr addrspace(1) %out) {
11814 entry:
11815 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") unordered, align 4
11816 ret void
11817 }
11819 define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
11820 ; GFX6-LABEL: global_workgroup_one_as_monotonic_store:
11821 ; GFX6: ; %bb.0: ; %entry
11822 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
11823 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11824 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11825 ; GFX6-NEXT: s_mov_b32 s11, s5
11826 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11827 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
11828 ; GFX6-NEXT: s_mov_b32 s10, -1
11829 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11830 ; GFX6-NEXT: s_mov_b32 s5, s11
11831 ; GFX6-NEXT: s_mov_b32 s6, s10
11832 ; GFX6-NEXT: s_mov_b32 s7, s9
11833 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
11834 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
11835 ; GFX6-NEXT: s_endpgm
11836 ;
11837 ; GFX7-LABEL: global_workgroup_one_as_monotonic_store:
11838 ; GFX7: ; %bb.0: ; %entry
11839 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
11840 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
11841 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
11842 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
11843 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
11844 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
11845 ; GFX7-NEXT: flat_store_dword v[0:1], v2
11846 ; GFX7-NEXT: s_endpgm
11847 ;
11848 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_store:
11849 ; GFX10-WGP: ; %bb.0: ; %entry
11850 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
11851 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
11852 ; GFX10-WGP-NEXT: s_nop 0
11853 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11854 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
11855 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
11856 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
11857 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
11858 ; GFX10-WGP-NEXT: s_endpgm
11859 ;
11860 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_store:
11861 ; GFX10-CU: ; %bb.0: ; %entry
11862 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
11863 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
11864 ; GFX10-CU-NEXT: s_nop 0
11865 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11866 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
11867 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
11868 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
11869 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
11870 ; GFX10-CU-NEXT: s_endpgm
11871 ;
11872 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_store:
11873 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
11874 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
11875 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
11876 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
11877 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
11878 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
11879 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
11880 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
11881 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
11882 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
11883 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
11884 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
11885 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
11886 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
11887 ; SKIP-CACHE-INV-NEXT: s_endpgm
11888 ;
11889 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_store:
11890 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
11891 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11892 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
11893 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
11894 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11895 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11896 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11897 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
11898 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11899 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
11900 ;
11901 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_store:
11902 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
11903 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
11904 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
11905 ; GFX90A-TGSPLIT-NEXT: s_nop 0
11906 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
11907 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11908 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11909 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
11910 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
11911 ; GFX90A-TGSPLIT-NEXT: s_endpgm
11912 ;
11913 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_store:
11914 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
11915 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11916 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
11917 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
11918 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11919 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11920 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11921 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
11922 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11923 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
11924 ;
11925 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_store:
11926 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
11927 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
11928 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
11929 ; GFX940-TGSPLIT-NEXT: s_nop 0
11930 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
11931 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
11932 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
11933 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
11934 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
11935 ; GFX940-TGSPLIT-NEXT: s_endpgm
11936 ;
11937 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_store:
11938 ; GFX11-WGP: ; %bb.0: ; %entry
11939 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11940 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
11941 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11942 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
11943 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
11944 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
11945 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
11946 ; GFX11-WGP-NEXT: s_endpgm
11947 ;
11948 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_store:
11949 ; GFX11-CU: ; %bb.0: ; %entry
11950 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11951 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
11952 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11953 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
11954 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
11955 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
11956 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11957 ; GFX11-CU-NEXT: s_endpgm
11958 ;
11959 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_store:
11960 ; GFX12-WGP: ; %bb.0: ; %entry
11961 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
11962 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
11963 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11964 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
11965 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
11966 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
11967 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
11968 ; GFX12-WGP-NEXT: s_endpgm
11969 ;
11970 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_store:
11971 ; GFX12-CU: ; %bb.0: ; %entry
11972 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
11973 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
11974 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
11975 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
11976 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
11977 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
11978 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
11979 ; GFX12-CU-NEXT: s_endpgm
11980 i32 %in, ptr addrspace(1) %out) {
11981 entry:
11982 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") monotonic, align 4
11983 ret void
11984 }
11986 define amdgpu_kernel void @global_workgroup_one_as_release_store(
11987 ; GFX6-LABEL: global_workgroup_one_as_release_store:
11988 ; GFX6: ; %bb.0: ; %entry
11989 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
11990 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
11991 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
11992 ; GFX6-NEXT: s_mov_b32 s11, s5
11993 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
11994 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
11995 ; GFX6-NEXT: s_mov_b32 s10, -1
11996 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
11997 ; GFX6-NEXT: s_mov_b32 s5, s11
11998 ; GFX6-NEXT: s_mov_b32 s6, s10
11999 ; GFX6-NEXT: s_mov_b32 s7, s9
12000 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12001 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
12002 ; GFX6-NEXT: s_endpgm
12003 ;
12004 ; GFX7-LABEL: global_workgroup_one_as_release_store:
12005 ; GFX7: ; %bb.0: ; %entry
12006 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
12007 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
12008 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12009 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12010 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12011 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12012 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12013 ; GFX7-NEXT: s_endpgm
12014 ;
12015 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_store:
12016 ; GFX10-WGP: ; %bb.0: ; %entry
12017 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
12018 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
12019 ; GFX10-WGP-NEXT: s_nop 0
12020 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12021 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12022 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12023 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12024 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12025 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12026 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
12027 ; GFX10-WGP-NEXT: s_endpgm
12028 ;
12029 ; GFX10-CU-LABEL: global_workgroup_one_as_release_store:
12030 ; GFX10-CU: ; %bb.0: ; %entry
12031 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
12032 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
12033 ; GFX10-CU-NEXT: s_nop 0
12034 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12035 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12036 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12037 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12038 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
12039 ; GFX10-CU-NEXT: s_endpgm
12040 ;
12041 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_store:
12042 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12043 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
12044 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
12045 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12046 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12047 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12048 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12049 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12050 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12051 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12052 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12053 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12054 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12055 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
12056 ; SKIP-CACHE-INV-NEXT: s_endpgm
12057 ;
12058 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_store:
12059 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12060 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12061 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
12062 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12063 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12064 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12065 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12066 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12067 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
12068 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12069 ;
12070 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_store:
12071 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12072 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12073 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
12074 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12075 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12076 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12077 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12078 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12079 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12080 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
12081 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12082 ;
12083 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_store:
12084 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12085 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12086 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
12087 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12088 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
12089 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12090 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12091 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12092 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
12093 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12094 ;
12095 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_store:
12096 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12097 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12098 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
12099 ; GFX940-TGSPLIT-NEXT: s_nop 0
12100 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
12101 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12102 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12103 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12104 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12105 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
12106 ; GFX940-TGSPLIT-NEXT: s_endpgm
12107 ;
12108 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_store:
12109 ; GFX11-WGP: ; %bb.0: ; %entry
12110 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12111 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
12112 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12113 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12114 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12115 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12116 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12117 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12118 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
12119 ; GFX11-WGP-NEXT: s_endpgm
12120 ;
12121 ; GFX11-CU-LABEL: global_workgroup_one_as_release_store:
12122 ; GFX11-CU: ; %bb.0: ; %entry
12123 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12124 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
12125 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12126 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12127 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12128 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12129 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
12130 ; GFX11-CU-NEXT: s_endpgm
12131 ;
12132 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_store:
12133 ; GFX12-WGP: ; %bb.0: ; %entry
12134 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12135 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
12136 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12137 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12138 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12139 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12140 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12141 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12142 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12143 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12144 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12145 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
12146 ; GFX12-WGP-NEXT: s_endpgm
12147 ;
12148 ; GFX12-CU-LABEL: global_workgroup_one_as_release_store:
12149 ; GFX12-CU: ; %bb.0: ; %entry
12150 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12151 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
12152 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12153 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
12154 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12155 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
12156 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
12157 ; GFX12-CU-NEXT: s_endpgm
12158 i32 %in, ptr addrspace(1) %out) {
12159 entry:
12160 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") release, align 4
12161 ret void
12162 }
12164 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
12165 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_store:
12166 ; GFX6: ; %bb.0: ; %entry
12167 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
12168 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
12169 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
12170 ; GFX6-NEXT: s_mov_b32 s11, s5
12171 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
12172 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
12173 ; GFX6-NEXT: s_mov_b32 s10, -1
12174 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
12175 ; GFX6-NEXT: s_mov_b32 s5, s11
12176 ; GFX6-NEXT: s_mov_b32 s6, s10
12177 ; GFX6-NEXT: s_mov_b32 s7, s9
12178 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12179 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
12180 ; GFX6-NEXT: s_endpgm
12181 ;
12182 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_store:
12183 ; GFX7: ; %bb.0: ; %entry
12184 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
12185 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[6:7], 0x2
12186 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12187 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12188 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12189 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12190 ; GFX7-NEXT: flat_store_dword v[0:1], v2
12191 ; GFX7-NEXT: s_endpgm
12192 ;
12193 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_store:
12194 ; GFX10-WGP: ; %bb.0: ; %entry
12195 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
12196 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
12197 ; GFX10-WGP-NEXT: s_nop 0
12198 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12199 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12200 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12201 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12202 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12203 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12204 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
12205 ; GFX10-WGP-NEXT: s_endpgm
12206 ;
12207 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_store:
12208 ; GFX10-CU: ; %bb.0: ; %entry
12209 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
12210 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
12211 ; GFX10-CU-NEXT: s_nop 0
12212 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12213 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12214 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12215 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12216 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
12217 ; GFX10-CU-NEXT: s_endpgm
12218 ;
12219 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_store:
12220 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12221 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
12222 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
12223 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12224 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12225 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12226 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12227 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12228 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12229 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12230 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12231 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12232 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12233 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
12234 ; SKIP-CACHE-INV-NEXT: s_endpgm
12235 ;
12236 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_store:
12237 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12238 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12239 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
12240 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12241 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12242 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12243 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12244 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12245 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
12246 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12247 ;
12248 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_store:
12249 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12250 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
12251 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
12252 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12253 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
12254 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12255 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12256 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12257 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12258 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
12259 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12260 ;
12261 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_store:
12262 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12263 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12264 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
12265 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12266 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
12267 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12268 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12269 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12270 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
12271 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12272 ;
12273 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_store:
12274 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12275 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
12276 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
12277 ; GFX940-TGSPLIT-NEXT: s_nop 0
12278 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
12279 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12280 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12281 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12282 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12283 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
12284 ; GFX940-TGSPLIT-NEXT: s_endpgm
12285 ;
12286 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_store:
12287 ; GFX11-WGP: ; %bb.0: ; %entry
12288 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12289 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
12290 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12291 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12292 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12293 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12294 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12295 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12296 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
12297 ; GFX11-WGP-NEXT: s_endpgm
12298 ;
12299 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_store:
12300 ; GFX11-CU: ; %bb.0: ; %entry
12301 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12302 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
12303 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12304 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12305 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12306 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12307 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
12308 ; GFX11-CU-NEXT: s_endpgm
12309 ;
12310 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_store:
12311 ; GFX12-WGP: ; %bb.0: ; %entry
12312 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
12313 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
12314 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12315 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12316 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12317 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12318 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12319 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12320 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12321 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12322 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12323 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
12324 ; GFX12-WGP-NEXT: s_endpgm
12325 ;
12326 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_store:
12327 ; GFX12-CU: ; %bb.0: ; %entry
12328 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
12329 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
12330 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
12331 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
12332 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12333 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
12334 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
12335 ; GFX12-CU-NEXT: s_endpgm
12336 i32 %in, ptr addrspace(1) %out) {
12337 entry:
12338 store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") seq_cst, align 4
12339 ret void
12340 }
12342 define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
12343 ; GFX6-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12344 ; GFX6: ; %bb.0: ; %entry
12345 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12346 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
12347 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
12348 ; GFX6-NEXT: s_mov_b32 s11, s5
12349 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
12350 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
12351 ; GFX6-NEXT: s_mov_b32 s10, -1
12352 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
12353 ; GFX6-NEXT: s_mov_b32 s5, s11
12354 ; GFX6-NEXT: s_mov_b32 s6, s10
12355 ; GFX6-NEXT: s_mov_b32 s7, s9
12356 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12357 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
12358 ; GFX6-NEXT: s_endpgm
12359 ;
12360 ; GFX7-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12361 ; GFX7: ; %bb.0: ; %entry
12362 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12363 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12364 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12365 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12366 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12367 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12368 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12369 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12370 ; GFX7-NEXT: s_endpgm
12371 ;
12372 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12373 ; GFX10-WGP: ; %bb.0: ; %entry
12374 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12375 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12376 ; GFX10-WGP-NEXT: s_nop 0
12377 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12378 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12379 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12380 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
12381 ; GFX10-WGP-NEXT: s_endpgm
12382 ;
12383 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12384 ; GFX10-CU: ; %bb.0: ; %entry
12385 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12386 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12387 ; GFX10-CU-NEXT: s_nop 0
12388 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12389 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12390 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12391 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
12392 ; GFX10-CU-NEXT: s_endpgm
12393 ;
12394 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12395 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12396 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12397 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
12398 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12399 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12400 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12401 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12402 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12403 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12404 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12405 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12406 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12407 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12408 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
12409 ; SKIP-CACHE-INV-NEXT: s_endpgm
12410 ;
12411 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12412 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12413 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12414 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12415 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12416 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12417 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12418 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12419 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12420 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12421 ;
12422 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12423 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12424 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12425 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12426 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12427 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12428 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12429 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12430 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12431 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12432 ;
12433 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12434 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12435 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12436 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12437 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12438 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12439 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12440 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12441 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12442 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12443 ;
12444 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12445 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12446 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12447 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12448 ; GFX940-TGSPLIT-NEXT: s_nop 0
12449 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12450 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12451 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12452 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12453 ; GFX940-TGSPLIT-NEXT: s_endpgm
12454 ;
12455 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12456 ; GFX11-WGP: ; %bb.0: ; %entry
12457 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12458 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12459 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12460 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12461 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12462 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12463 ; GFX11-WGP-NEXT: s_endpgm
12464 ;
12465 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12466 ; GFX11-CU: ; %bb.0: ; %entry
12467 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12468 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12469 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12470 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12471 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12472 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12473 ; GFX11-CU-NEXT: s_endpgm
12474 ;
12475 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12476 ; GFX12-WGP: ; %bb.0: ; %entry
12477 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12478 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12479 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12480 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12481 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12482 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
12483 ; GFX12-WGP-NEXT: s_endpgm
12484 ;
12485 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
12486 ; GFX12-CU: ; %bb.0: ; %entry
12487 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
12488 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12489 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12490 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12491 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
12492 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12493 ; GFX12-CU-NEXT: s_endpgm
12494 ptr addrspace(1) %out, i32 %in) {
12495 entry:
12496 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") monotonic
12497 ret void
12498 }
12500 define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
12501 ; GFX6-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12502 ; GFX6: ; %bb.0: ; %entry
12503 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12504 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
12505 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
12506 ; GFX6-NEXT: s_mov_b32 s11, s5
12507 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
12508 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
12509 ; GFX6-NEXT: s_mov_b32 s10, -1
12510 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
12511 ; GFX6-NEXT: s_mov_b32 s5, s11
12512 ; GFX6-NEXT: s_mov_b32 s6, s10
12513 ; GFX6-NEXT: s_mov_b32 s7, s9
12514 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12515 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
12516 ; GFX6-NEXT: s_endpgm
12517 ;
12518 ; GFX7-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12519 ; GFX7: ; %bb.0: ; %entry
12520 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12521 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12522 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12523 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12524 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12525 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12526 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12527 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12528 ; GFX7-NEXT: s_endpgm
12529 ;
12530 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12531 ; GFX10-WGP: ; %bb.0: ; %entry
12532 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12533 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12534 ; GFX10-WGP-NEXT: s_nop 0
12535 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12536 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12537 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12538 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
12539 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12540 ; GFX10-WGP-NEXT: buffer_gl0_inv
12541 ; GFX10-WGP-NEXT: s_endpgm
12542 ;
12543 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12544 ; GFX10-CU: ; %bb.0: ; %entry
12545 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12546 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12547 ; GFX10-CU-NEXT: s_nop 0
12548 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12549 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12550 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12551 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
12552 ; GFX10-CU-NEXT: s_endpgm
12553 ;
12554 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12555 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12556 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12557 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
12558 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12559 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12560 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12561 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12562 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12563 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12564 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12565 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12566 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12567 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12568 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
12569 ; SKIP-CACHE-INV-NEXT: s_endpgm
12570 ;
12571 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12572 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12573 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12574 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12575 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12576 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12577 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12578 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12579 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12580 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12581 ;
12582 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12583 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12584 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12585 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12586 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12587 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12588 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12589 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12590 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12591 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12592 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12593 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12594 ;
12595 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12596 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12597 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12598 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12599 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12600 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12601 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12602 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12603 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12604 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12605 ;
12606 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12607 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12608 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12609 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12610 ; GFX940-TGSPLIT-NEXT: s_nop 0
12611 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12612 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12613 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12614 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12615 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12616 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12617 ; GFX940-TGSPLIT-NEXT: s_endpgm
12618 ;
12619 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12620 ; GFX11-WGP: ; %bb.0: ; %entry
12621 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12622 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12623 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12624 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12625 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12626 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12627 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12628 ; GFX11-WGP-NEXT: buffer_gl0_inv
12629 ; GFX11-WGP-NEXT: s_endpgm
12630 ;
12631 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12632 ; GFX11-CU: ; %bb.0: ; %entry
12633 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12634 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12635 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12636 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12637 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12638 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12639 ; GFX11-CU-NEXT: s_endpgm
12640 ;
12641 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12642 ; GFX12-WGP: ; %bb.0: ; %entry
12643 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12644 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12645 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12646 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12647 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12648 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
12649 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12650 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12651 ; GFX12-WGP-NEXT: s_endpgm
12652 ;
12653 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_atomicrmw:
12654 ; GFX12-CU: ; %bb.0: ; %entry
12655 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
12656 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12657 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12658 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12659 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
12660 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12661 ; GFX12-CU-NEXT: s_endpgm
12662 ptr addrspace(1) %out, i32 %in) {
12663 entry:
12664 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
12665 ret void
12666 }
12668 define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
12669 ; GFX6-LABEL: global_workgroup_one_as_release_atomicrmw:
12670 ; GFX6: ; %bb.0: ; %entry
12671 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12672 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
12673 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
12674 ; GFX6-NEXT: s_mov_b32 s11, s5
12675 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
12676 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
12677 ; GFX6-NEXT: s_mov_b32 s10, -1
12678 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
12679 ; GFX6-NEXT: s_mov_b32 s5, s11
12680 ; GFX6-NEXT: s_mov_b32 s6, s10
12681 ; GFX6-NEXT: s_mov_b32 s7, s9
12682 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12683 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
12684 ; GFX6-NEXT: s_endpgm
12685 ;
12686 ; GFX7-LABEL: global_workgroup_one_as_release_atomicrmw:
12687 ; GFX7: ; %bb.0: ; %entry
12688 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12689 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12690 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12691 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12692 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12693 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12694 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12695 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12696 ; GFX7-NEXT: s_endpgm
12697 ;
12698 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_atomicrmw:
12699 ; GFX10-WGP: ; %bb.0: ; %entry
12700 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12701 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12702 ; GFX10-WGP-NEXT: s_nop 0
12703 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12704 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12705 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12706 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12707 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12708 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
12709 ; GFX10-WGP-NEXT: s_endpgm
12710 ;
12711 ; GFX10-CU-LABEL: global_workgroup_one_as_release_atomicrmw:
12712 ; GFX10-CU: ; %bb.0: ; %entry
12713 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12714 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12715 ; GFX10-CU-NEXT: s_nop 0
12716 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12717 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12718 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12719 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
12720 ; GFX10-CU-NEXT: s_endpgm
12721 ;
12722 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_atomicrmw:
12723 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12724 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12725 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
12726 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12727 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12728 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12729 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12730 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12731 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12732 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12733 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12734 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12735 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12736 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
12737 ; SKIP-CACHE-INV-NEXT: s_endpgm
12738 ;
12739 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_atomicrmw:
12740 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12741 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12742 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12743 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12744 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12745 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12746 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12747 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12748 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12749 ;
12750 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_atomicrmw:
12751 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12752 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12753 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12754 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12755 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12756 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12757 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12758 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12759 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12760 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12761 ;
12762 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_atomicrmw:
12763 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12764 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12765 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12766 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12767 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12768 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12769 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12770 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12771 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12772 ;
12773 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_atomicrmw:
12774 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12775 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12776 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12777 ; GFX940-TGSPLIT-NEXT: s_nop 0
12778 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12779 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12780 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12781 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12782 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12783 ; GFX940-TGSPLIT-NEXT: s_endpgm
12784 ;
12785 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_atomicrmw:
12786 ; GFX11-WGP: ; %bb.0: ; %entry
12787 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12788 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12789 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12790 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12791 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12792 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12793 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12794 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12795 ; GFX11-WGP-NEXT: s_endpgm
12796 ;
12797 ; GFX11-CU-LABEL: global_workgroup_one_as_release_atomicrmw:
12798 ; GFX11-CU: ; %bb.0: ; %entry
12799 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12800 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12801 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12802 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12803 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12804 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12805 ; GFX11-CU-NEXT: s_endpgm
12806 ;
12807 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_atomicrmw:
12808 ; GFX12-WGP: ; %bb.0: ; %entry
12809 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12810 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12811 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12812 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12813 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12814 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12815 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12816 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12817 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12818 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12819 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
12820 ; GFX12-WGP-NEXT: s_endpgm
12821 ;
12822 ; GFX12-CU-LABEL: global_workgroup_one_as_release_atomicrmw:
12823 ; GFX12-CU: ; %bb.0: ; %entry
12824 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
12825 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12826 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12827 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
12828 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
12829 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12830 ; GFX12-CU-NEXT: s_endpgm
12831 ptr addrspace(1) %out, i32 %in) {
12832 entry:
12833 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") release
12834 ret void
12835 }
12837 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
12838 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12839 ; GFX6: ; %bb.0: ; %entry
12840 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12841 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
12842 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
12843 ; GFX6-NEXT: s_mov_b32 s11, s5
12844 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
12845 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
12846 ; GFX6-NEXT: s_mov_b32 s10, -1
12847 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
12848 ; GFX6-NEXT: s_mov_b32 s5, s11
12849 ; GFX6-NEXT: s_mov_b32 s6, s10
12850 ; GFX6-NEXT: s_mov_b32 s7, s9
12851 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
12852 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
12853 ; GFX6-NEXT: s_endpgm
12854 ;
12855 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12856 ; GFX7: ; %bb.0: ; %entry
12857 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
12858 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
12859 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
12860 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
12861 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
12862 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
12863 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
12864 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
12865 ; GFX7-NEXT: s_endpgm
12866 ;
12867 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12868 ; GFX10-WGP: ; %bb.0: ; %entry
12869 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
12870 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12871 ; GFX10-WGP-NEXT: s_nop 0
12872 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
12873 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
12874 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
12875 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
12876 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12877 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
12878 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12879 ; GFX10-WGP-NEXT: buffer_gl0_inv
12880 ; GFX10-WGP-NEXT: s_endpgm
12881 ;
12882 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12883 ; GFX10-CU: ; %bb.0: ; %entry
12884 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
12885 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12886 ; GFX10-CU-NEXT: s_nop 0
12887 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
12888 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
12889 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
12890 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
12891 ; GFX10-CU-NEXT: s_endpgm
12892 ;
12893 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12894 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
12895 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12896 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
12897 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
12898 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
12899 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
12900 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
12901 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
12902 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
12903 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
12904 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
12905 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
12906 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
12907 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
12908 ; SKIP-CACHE-INV-NEXT: s_endpgm
12909 ;
12910 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12911 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
12912 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12913 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12914 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
12915 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12916 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12917 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12918 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12919 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
12920 ;
12921 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12922 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
12923 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12924 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
12925 ; GFX90A-TGSPLIT-NEXT: s_nop 0
12926 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
12927 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12928 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
12929 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12930 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
12931 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12932 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
12933 ; GFX90A-TGSPLIT-NEXT: s_endpgm
12934 ;
12935 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12936 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
12937 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12938 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12939 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
12940 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12941 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12942 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12943 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12944 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
12945 ;
12946 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12947 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
12948 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
12949 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
12950 ; GFX940-TGSPLIT-NEXT: s_nop 0
12951 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
12952 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
12953 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
12954 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12955 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
12956 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
12957 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
12958 ; GFX940-TGSPLIT-NEXT: s_endpgm
12959 ;
12960 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12961 ; GFX11-WGP: ; %bb.0: ; %entry
12962 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
12963 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12964 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12965 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
12966 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
12967 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
12968 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12969 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12970 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
12971 ; GFX11-WGP-NEXT: buffer_gl0_inv
12972 ; GFX11-WGP-NEXT: s_endpgm
12973 ;
12974 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12975 ; GFX11-CU: ; %bb.0: ; %entry
12976 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
12977 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12978 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
12979 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
12980 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
12981 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
12982 ; GFX11-CU-NEXT: s_endpgm
12983 ;
12984 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
12985 ; GFX12-WGP: ; %bb.0: ; %entry
12986 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
12987 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
12988 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
12989 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
12990 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
12991 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
12992 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
12993 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
12994 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
12995 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12996 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
12997 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
12998 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
12999 ; GFX12-WGP-NEXT: s_endpgm
13000 ;
13001 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
13002 ; GFX12-CU: ; %bb.0: ; %entry
13003 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
13004 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13005 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13006 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13007 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
13008 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
13009 ; GFX12-CU-NEXT: s_endpgm
13010 ptr addrspace(1) %out, i32 %in) {
13011 entry:
13012 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
13013 ret void
13014 }
13016 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
13017 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13018 ; GFX6: ; %bb.0: ; %entry
13019 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13020 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
13021 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
13022 ; GFX6-NEXT: s_mov_b32 s11, s5
13023 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
13024 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
13025 ; GFX6-NEXT: s_mov_b32 s10, -1
13026 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
13027 ; GFX6-NEXT: s_mov_b32 s5, s11
13028 ; GFX6-NEXT: s_mov_b32 s6, s10
13029 ; GFX6-NEXT: s_mov_b32 s7, s9
13030 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
13031 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
13032 ; GFX6-NEXT: s_endpgm
13033 ;
13034 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13035 ; GFX7: ; %bb.0: ; %entry
13036 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13037 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
13038 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
13039 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13040 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13041 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
13042 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
13043 ; GFX7-NEXT: flat_atomic_swap v[0:1], v2
13044 ; GFX7-NEXT: s_endpgm
13045 ;
13046 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13047 ; GFX10-WGP: ; %bb.0: ; %entry
13048 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
13049 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13050 ; GFX10-WGP-NEXT: s_nop 0
13051 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
13052 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13053 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
13054 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13055 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13056 ; GFX10-WGP-NEXT: global_atomic_swap v0, v1, s[4:5]
13057 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13058 ; GFX10-WGP-NEXT: buffer_gl0_inv
13059 ; GFX10-WGP-NEXT: s_endpgm
13060 ;
13061 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13062 ; GFX10-CU: ; %bb.0: ; %entry
13063 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
13064 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13065 ; GFX10-CU-NEXT: s_nop 0
13066 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
13067 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13068 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
13069 ; GFX10-CU-NEXT: global_atomic_swap v0, v1, s[4:5]
13070 ; GFX10-CU-NEXT: s_endpgm
13071 ;
13072 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13073 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13074 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13075 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
13076 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13077 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
13078 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
13079 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
13080 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
13081 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
13082 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
13083 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
13084 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
13085 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
13086 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
13087 ; SKIP-CACHE-INV-NEXT: s_endpgm
13088 ;
13089 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13090 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13091 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13092 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13093 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
13094 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13095 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13096 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13097 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
13098 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13099 ;
13100 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13101 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13102 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13103 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13104 ; GFX90A-TGSPLIT-NEXT: s_nop 0
13105 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13106 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13107 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13108 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13109 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[4:5]
13110 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13111 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13112 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13113 ;
13114 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13115 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13116 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13117 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13118 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
13119 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13120 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13121 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13122 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
13123 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13124 ;
13125 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13126 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13127 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13128 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13129 ; GFX940-TGSPLIT-NEXT: s_nop 0
13130 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13131 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13132 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13133 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13134 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v0, v1, s[0:1]
13135 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13136 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13137 ; GFX940-TGSPLIT-NEXT: s_endpgm
13138 ;
13139 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13140 ; GFX11-WGP: ; %bb.0: ; %entry
13141 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
13142 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13143 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13144 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13145 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
13146 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13147 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13148 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
13149 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13150 ; GFX11-WGP-NEXT: buffer_gl0_inv
13151 ; GFX11-WGP-NEXT: s_endpgm
13152 ;
13153 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13154 ; GFX11-CU: ; %bb.0: ; %entry
13155 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
13156 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13157 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13158 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13159 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
13160 ; GFX11-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
13161 ; GFX11-CU-NEXT: s_endpgm
13162 ;
13163 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13164 ; GFX12-WGP: ; %bb.0: ; %entry
13165 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
13166 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13167 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13168 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13169 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
13170 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
13171 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13172 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13173 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13174 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13175 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SE
13176 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13177 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13178 ; GFX12-WGP-NEXT: s_endpgm
13179 ;
13180 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
13181 ; GFX12-CU: ; %bb.0: ; %entry
13182 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
13183 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13184 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13185 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13186 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
13187 ; GFX12-CU-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
13188 ; GFX12-CU-NEXT: s_endpgm
13189 ptr addrspace(1) %out, i32 %in) {
13190 entry:
13191 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
13192 ret void
13193 }
13195 define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
13196 ; GFX6-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13197 ; GFX6: ; %bb.0: ; %entry
13198 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13199 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
13200 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
13201 ; GFX6-NEXT: s_mov_b32 s11, s5
13202 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
13203 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
13204 ; GFX6-NEXT: s_mov_b32 s10, -1
13205 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
13206 ; GFX6-NEXT: s_mov_b32 s5, s11
13207 ; GFX6-NEXT: s_mov_b32 s6, s10
13208 ; GFX6-NEXT: s_mov_b32 s7, s9
13209 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
13210 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
13211 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13212 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
13213 ; GFX6-NEXT: s_endpgm
13214 ;
13215 ; GFX7-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13216 ; GFX7: ; %bb.0: ; %entry
13217 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13218 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
13219 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13220 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13221 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13222 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13223 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13224 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13225 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13226 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13227 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13228 ; GFX7-NEXT: s_endpgm
13229 ;
13230 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13231 ; GFX10-WGP: ; %bb.0: ; %entry
13232 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
13233 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13234 ; GFX10-WGP-NEXT: s_nop 0
13235 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
13236 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
13238 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13239 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13240 ; GFX10-WGP-NEXT: buffer_gl0_inv
13241 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
13242 ; GFX10-WGP-NEXT: s_endpgm
13243 ;
13244 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13245 ; GFX10-CU: ; %bb.0: ; %entry
13246 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
13247 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13248 ; GFX10-CU-NEXT: s_nop 0
13249 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
13250 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13251 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
13252 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13253 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13254 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
13255 ; GFX10-CU-NEXT: s_endpgm
13256 ;
13257 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13258 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13259 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13260 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
13261 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13262 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
13263 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
13264 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
13265 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
13266 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
13267 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
13268 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
13269 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
13270 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
13271 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
13272 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13273 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
13274 ; SKIP-CACHE-INV-NEXT: s_endpgm
13275 ;
13276 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13277 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13278 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13279 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13280 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
13281 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13282 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13283 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13284 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13285 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13286 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13287 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13288 ;
13289 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13290 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13291 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13292 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13293 ; GFX90A-TGSPLIT-NEXT: s_nop 0
13294 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13295 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13296 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13297 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13298 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13299 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13300 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13301 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13302 ;
13303 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13304 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13305 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13306 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13307 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
13308 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13309 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13310 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13311 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13312 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13313 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13314 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13315 ;
13316 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13317 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13318 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13319 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13320 ; GFX940-TGSPLIT-NEXT: s_nop 0
13321 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13322 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13323 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13324 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13325 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13326 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13327 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13328 ; GFX940-TGSPLIT-NEXT: s_endpgm
13329 ;
13330 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13331 ; GFX11-WGP: ; %bb.0: ; %entry
13332 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
13333 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13334 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13335 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13336 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
13337 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13338 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13339 ; GFX11-WGP-NEXT: buffer_gl0_inv
13340 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13341 ; GFX11-WGP-NEXT: s_endpgm
13342 ;
13343 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13344 ; GFX11-CU: ; %bb.0: ; %entry
13345 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
13346 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13347 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13348 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13349 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
13350 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13351 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13352 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13353 ; GFX11-CU-NEXT: s_endpgm
13354 ;
13355 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13356 ; GFX12-WGP: ; %bb.0: ; %entry
13357 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
13358 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13359 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13360 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13361 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
13362 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
13363 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13364 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13365 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13366 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13367 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13368 ; GFX12-WGP-NEXT: s_endpgm
13369 ;
13370 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
13371 ; GFX12-CU: ; %bb.0: ; %entry
13372 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
13373 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13374 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13375 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13376 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
13377 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
13378 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13379 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13380 ; GFX12-CU-NEXT: s_endpgm
13381 ptr addrspace(1) %out, i32 %in) {
13382 entry:
13383 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
13384 store i32 %val, ptr addrspace(1) %out, align 4
13385 ret void
13386 }
13388 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
13389 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13390 ; GFX6: ; %bb.0: ; %entry
13391 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13392 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
13393 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
13394 ; GFX6-NEXT: s_mov_b32 s11, s5
13395 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
13396 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
13397 ; GFX6-NEXT: s_mov_b32 s10, -1
13398 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
13399 ; GFX6-NEXT: s_mov_b32 s5, s11
13400 ; GFX6-NEXT: s_mov_b32 s6, s10
13401 ; GFX6-NEXT: s_mov_b32 s7, s9
13402 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
13403 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
13404 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13405 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
13406 ; GFX6-NEXT: s_endpgm
13407 ;
13408 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13409 ; GFX7: ; %bb.0: ; %entry
13410 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13411 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
13412 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13413 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13414 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13415 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13416 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13417 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13418 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13419 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13420 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13421 ; GFX7-NEXT: s_endpgm
13422 ;
13423 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13424 ; GFX10-WGP: ; %bb.0: ; %entry
13425 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
13426 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13427 ; GFX10-WGP-NEXT: s_nop 0
13428 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
13429 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13430 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
13431 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13432 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13433 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13434 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13435 ; GFX10-WGP-NEXT: buffer_gl0_inv
13436 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
13437 ; GFX10-WGP-NEXT: s_endpgm
13438 ;
13439 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13440 ; GFX10-CU: ; %bb.0: ; %entry
13441 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
13442 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13443 ; GFX10-CU-NEXT: s_nop 0
13444 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
13445 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13446 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
13447 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13448 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13449 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
13450 ; GFX10-CU-NEXT: s_endpgm
13451 ;
13452 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13453 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13454 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13455 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
13456 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13457 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
13458 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
13459 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
13460 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
13461 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
13462 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
13463 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
13464 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
13465 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
13466 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
13467 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13468 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
13469 ; SKIP-CACHE-INV-NEXT: s_endpgm
13470 ;
13471 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13472 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13473 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13474 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13475 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
13476 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13477 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13478 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13479 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13480 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13481 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13482 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13483 ;
13484 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13485 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13486 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13487 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13488 ; GFX90A-TGSPLIT-NEXT: s_nop 0
13489 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13490 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13491 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13492 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13493 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13494 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13495 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13496 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13497 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13498 ;
13499 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13500 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13501 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13502 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13503 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
13504 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13505 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13506 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13507 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13508 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13509 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13510 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13511 ;
13512 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13513 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13514 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13515 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13516 ; GFX940-TGSPLIT-NEXT: s_nop 0
13517 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13518 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13519 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13520 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13521 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13522 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13523 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13524 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13525 ; GFX940-TGSPLIT-NEXT: s_endpgm
13526 ;
13527 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13528 ; GFX11-WGP: ; %bb.0: ; %entry
13529 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
13530 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13531 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13532 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13533 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
13534 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13535 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13536 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13537 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13538 ; GFX11-WGP-NEXT: buffer_gl0_inv
13539 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13540 ; GFX11-WGP-NEXT: s_endpgm
13541 ;
13542 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13543 ; GFX11-CU: ; %bb.0: ; %entry
13544 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
13545 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13546 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13547 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13548 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
13549 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13550 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13551 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13552 ; GFX11-CU-NEXT: s_endpgm
13553 ;
13554 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13555 ; GFX12-WGP: ; %bb.0: ; %entry
13556 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
13557 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13558 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13559 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13560 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
13561 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
13562 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13563 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13564 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13565 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13566 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
13567 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13568 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13569 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13570 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13571 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13572 ; GFX12-WGP-NEXT: s_endpgm
13573 ;
13574 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
13575 ; GFX12-CU: ; %bb.0: ; %entry
13576 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
13577 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13578 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13579 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13580 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
13581 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
13582 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13583 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13584 ; GFX12-CU-NEXT: s_endpgm
13585 ptr addrspace(1) %out, i32 %in) {
13586 entry:
13587 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
13588 store i32 %val, ptr addrspace(1) %out, align 4
13589 ret void
13590 }
13592 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
13593 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13594 ; GFX6: ; %bb.0: ; %entry
13595 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13596 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x2
13597 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
13598 ; GFX6-NEXT: s_mov_b32 s11, s5
13599 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
13600 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
13601 ; GFX6-NEXT: s_mov_b32 s10, -1
13602 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
13603 ; GFX6-NEXT: s_mov_b32 s5, s11
13604 ; GFX6-NEXT: s_mov_b32 s6, s10
13605 ; GFX6-NEXT: s_mov_b32 s7, s9
13606 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
13607 ; GFX6-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 glc
13608 ; GFX6-NEXT: s_waitcnt vmcnt(0)
13609 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
13610 ; GFX6-NEXT: s_endpgm
13611 ;
13612 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13613 ; GFX7: ; %bb.0: ; %entry
13614 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13615 ; GFX7-NEXT: s_load_dword s6, s[6:7], 0x2
13616 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13617 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13618 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13619 ; GFX7-NEXT: v_mov_b32_e32 v2, s6
13620 ; GFX7-NEXT: flat_atomic_swap v2, v[0:1], v2 glc
13621 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13622 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13623 ; GFX7-NEXT: s_waitcnt vmcnt(0)
13624 ; GFX7-NEXT: flat_store_dword v[0:1], v2
13625 ; GFX7-NEXT: s_endpgm
13626 ;
13627 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13628 ; GFX10-WGP: ; %bb.0: ; %entry
13629 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
13630 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13631 ; GFX10-WGP-NEXT: s_nop 0
13632 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
13633 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13634 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
13635 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13636 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13637 ; GFX10-WGP-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13638 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
13639 ; GFX10-WGP-NEXT: buffer_gl0_inv
13640 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
13641 ; GFX10-WGP-NEXT: s_endpgm
13642 ;
13643 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13644 ; GFX10-CU: ; %bb.0: ; %entry
13645 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
13646 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13647 ; GFX10-CU-NEXT: s_nop 0
13648 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
13649 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13650 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
13651 ; GFX10-CU-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13652 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
13653 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
13654 ; GFX10-CU-NEXT: s_endpgm
13655 ;
13656 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13657 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13658 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13659 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x2
13660 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13661 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
13662 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
13663 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
13664 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
13665 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
13666 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
13667 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
13668 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
13669 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
13670 ; SKIP-CACHE-INV-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 glc
13671 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
13672 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
13673 ; SKIP-CACHE-INV-NEXT: s_endpgm
13674 ;
13675 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13676 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13677 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13678 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13679 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
13680 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13681 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13682 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13683 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13684 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13685 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13686 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13687 ;
13688 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13689 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13690 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13691 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13692 ; GFX90A-TGSPLIT-NEXT: s_nop 0
13693 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
13694 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13695 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13696 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13697 ; GFX90A-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[4:5] glc
13698 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13699 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
13700 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
13701 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13702 ;
13703 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13704 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13705 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13706 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13707 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
13708 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13709 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13710 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13711 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13712 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
13713 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13714 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13715 ;
13716 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13717 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13718 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13719 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13720 ; GFX940-TGSPLIT-NEXT: s_nop 0
13721 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
13722 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13723 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13724 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13725 ; GFX940-TGSPLIT-NEXT: global_atomic_swap v1, v0, v1, s[0:1] sc0
13726 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
13727 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
13728 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
13729 ; GFX940-TGSPLIT-NEXT: s_endpgm
13730 ;
13731 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13732 ; GFX11-WGP: ; %bb.0: ; %entry
13733 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
13734 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13735 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13736 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13737 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s2
13738 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13739 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
13740 ; GFX11-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13741 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
13742 ; GFX11-WGP-NEXT: buffer_gl0_inv
13743 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13744 ; GFX11-WGP-NEXT: s_endpgm
13745 ;
13746 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13747 ; GFX11-CU: ; %bb.0: ; %entry
13748 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
13749 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13750 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13751 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13752 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s2
13753 ; GFX11-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] glc
13754 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
13755 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13756 ; GFX11-CU-NEXT: s_endpgm
13757 ;
13758 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13759 ; GFX12-WGP: ; %bb.0: ; %entry
13760 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
13761 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13762 ; GFX12-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
13763 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13764 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s2
13765 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
13766 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13767 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13768 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13769 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
13770 ; GFX12-WGP-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE
13771 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
13772 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
13773 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
13774 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
13775 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
13776 ; GFX12-WGP-NEXT: s_endpgm
13777 ;
13778 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
13779 ; GFX12-CU: ; %bb.0: ; %entry
13780 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
13781 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
13782 ; GFX12-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
13783 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
13784 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s2
13785 ; GFX12-CU-NEXT: global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
13786 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
13787 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
13788 ; GFX12-CU-NEXT: s_endpgm
13789 ptr addrspace(1) %out, i32 %in) {
13790 entry:
13791 %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
13792 store i32 %val, ptr addrspace(1) %out, align 4
13793 ret void
13794 }
13796 define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
13797 ; GFX6-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13798 ; GFX6: ; %bb.0: ; %entry
13799 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
13800 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
13801 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
13802 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
13803 ; GFX6-NEXT: s_mov_b32 s12, s5
13804 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
13805 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
13806 ; GFX6-NEXT: s_mov_b32 s11, -1
13807 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
13808 ; GFX6-NEXT: s_mov_b32 s5, s12
13809 ; GFX6-NEXT: s_mov_b32 s6, s11
13810 ; GFX6-NEXT: s_mov_b32 s7, s10
13811 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
13812 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
13813 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
13814 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
13815 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
13816 ; GFX6-NEXT: s_endpgm
13817 ;
13818 ; GFX7-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13819 ; GFX7: ; %bb.0: ; %entry
13820 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
13821 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
13822 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
13823 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
13824 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
13825 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
13826 ; GFX7-NEXT: s_mov_b32 s4, s8
13827 ; GFX7-NEXT: s_mov_b32 s5, s9
13828 ; GFX7-NEXT: s_mov_b32 s9, s10
13829 ; GFX7-NEXT: s_mov_b32 s8, s11
13830 ; GFX7-NEXT: s_add_u32 s4, s4, s9
13831 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
13832 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
13833 ; GFX7-NEXT: s_mov_b32 s5, s8
13834 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
13835 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
13836 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13837 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
13838 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
13839 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
13840 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
13841 ; GFX7-NEXT: s_endpgm
13842 ;
13843 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13844 ; GFX10-WGP: ; %bb.0: ; %entry
13845 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
13846 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
13847 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13848 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
13849 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
13850 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
13851 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
13852 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
13853 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
13854 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
13855 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
13856 ; GFX10-WGP-NEXT: s_endpgm
13857 ;
13858 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13859 ; GFX10-CU: ; %bb.0: ; %entry
13860 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
13861 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
13862 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13863 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
13864 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
13865 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
13866 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
13867 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
13868 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
13869 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
13870 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
13871 ; GFX10-CU-NEXT: s_endpgm
13872 ;
13873 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13874 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
13875 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
13876 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
13877 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
13878 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
13879 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
13880 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
13881 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
13882 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
13883 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
13884 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
13885 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
13886 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
13887 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
13888 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
13889 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
13890 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
13891 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
13892 ; SKIP-CACHE-INV-NEXT: s_endpgm
13893 ;
13894 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13895 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
13896 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13897 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13898 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13899 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13900 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13901 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13902 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13903 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13904 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13905 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
13906 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
13907 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
13908 ;
13909 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13910 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
13911 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
13912 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13913 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
13914 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
13915 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
13916 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13917 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
13918 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
13919 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13920 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
13921 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
13922 ; GFX90A-TGSPLIT-NEXT: s_endpgm
13923 ;
13924 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13925 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
13926 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13927 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13928 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13929 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13930 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13931 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13932 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13933 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13934 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13935 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
13936 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
13937 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
13938 ;
13939 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13940 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
13941 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
13942 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
13943 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
13944 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
13945 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
13946 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
13947 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
13948 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
13949 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
13950 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
13951 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
13952 ; GFX940-TGSPLIT-NEXT: s_endpgm
13953 ;
13954 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13955 ; GFX11-WGP: ; %bb.0: ; %entry
13956 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13957 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
13958 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13959 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13960 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13961 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
13962 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
13963 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
13964 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
13965 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
13966 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
13967 ; GFX11-WGP-NEXT: s_endpgm
13968 ;
13969 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13970 ; GFX11-CU: ; %bb.0: ; %entry
13971 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
13972 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
13973 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13974 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
13975 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
13976 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
13977 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
13978 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
13979 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
13980 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
13981 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
13982 ; GFX11-CU-NEXT: s_endpgm
13983 ;
13984 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
13985 ; GFX12-WGP: ; %bb.0: ; %entry
13986 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
13987 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
13988 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
13989 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
13990 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
13991 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
13992 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
13993 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
13994 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
13995 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
13996 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
13997 ; GFX12-WGP-NEXT: s_endpgm
13998 ;
13999 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
14000 ; GFX12-CU: ; %bb.0: ; %entry
14001 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14002 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
14003 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14004 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14005 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14006 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14007 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
14008 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
14009 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14010 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
14011 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14012 ; GFX12-CU-NEXT: s_endpgm
14013 ptr addrspace(1) %out, i32 %in, i32 %old) {
14014 entry:
14015 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
14016 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
14017 ret void
14018 }
14020 define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
14021 ; GFX6-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14022 ; GFX6: ; %bb.0: ; %entry
14023 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
14024 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
14025 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
14026 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
14027 ; GFX6-NEXT: s_mov_b32 s12, s5
14028 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
14029 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
14030 ; GFX6-NEXT: s_mov_b32 s11, -1
14031 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
14032 ; GFX6-NEXT: s_mov_b32 s5, s12
14033 ; GFX6-NEXT: s_mov_b32 s6, s11
14034 ; GFX6-NEXT: s_mov_b32 s7, s10
14035 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
14036 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
14037 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14038 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14039 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
14040 ; GFX6-NEXT: s_endpgm
14041 ;
14042 ; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14043 ; GFX7: ; %bb.0: ; %entry
14044 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14045 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14046 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14047 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14048 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14049 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14050 ; GFX7-NEXT: s_mov_b32 s4, s8
14051 ; GFX7-NEXT: s_mov_b32 s5, s9
14052 ; GFX7-NEXT: s_mov_b32 s9, s10
14053 ; GFX7-NEXT: s_mov_b32 s8, s11
14054 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14055 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14056 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14057 ; GFX7-NEXT: s_mov_b32 s5, s8
14058 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14059 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14060 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14061 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14062 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14063 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14064 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14065 ; GFX7-NEXT: s_endpgm
14066 ;
14067 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14068 ; GFX10-WGP: ; %bb.0: ; %entry
14069 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
14070 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
14071 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14072 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
14073 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
14074 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14075 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
14076 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
14077 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14078 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
14079 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14080 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14081 ; GFX10-WGP-NEXT: buffer_gl0_inv
14082 ; GFX10-WGP-NEXT: s_endpgm
14083 ;
14084 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14085 ; GFX10-CU: ; %bb.0: ; %entry
14086 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
14087 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
14088 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14089 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
14090 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
14091 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14092 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
14093 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
14094 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14095 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
14096 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14097 ; GFX10-CU-NEXT: s_endpgm
14098 ;
14099 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14100 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14101 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
14102 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
14103 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
14104 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14105 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
14106 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
14107 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
14108 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
14109 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
14110 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
14111 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
14112 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
14113 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
14114 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
14115 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14116 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
14117 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
14118 ; SKIP-CACHE-INV-NEXT: s_endpgm
14119 ;
14120 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14121 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14122 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14123 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14124 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14125 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14126 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14127 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14128 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14129 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14130 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14131 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14132 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14133 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14134 ;
14135 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14136 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14137 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14138 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14139 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14140 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14141 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14142 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14143 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14144 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14145 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14146 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14147 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14148 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14149 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14150 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14151 ;
14152 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14153 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14154 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14155 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14156 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14157 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14158 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14159 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14160 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14161 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14162 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14163 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14164 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14165 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14166 ;
14167 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14168 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14169 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14170 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14171 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14172 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14173 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14174 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14175 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14176 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14177 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14178 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14179 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14180 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14181 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14182 ; GFX940-TGSPLIT-NEXT: s_endpgm
14183 ;
14184 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14185 ; GFX11-WGP: ; %bb.0: ; %entry
14186 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14187 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
14188 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14189 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14190 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14191 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14192 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
14193 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
14194 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14195 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
14196 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14197 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14198 ; GFX11-WGP-NEXT: buffer_gl0_inv
14199 ; GFX11-WGP-NEXT: s_endpgm
14200 ;
14201 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14202 ; GFX11-CU: ; %bb.0: ; %entry
14203 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14204 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
14205 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14206 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14207 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14208 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14209 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
14210 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
14211 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14212 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
14213 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14214 ; GFX11-CU-NEXT: s_endpgm
14215 ;
14216 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14217 ; GFX12-WGP: ; %bb.0: ; %entry
14218 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14219 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
14220 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14221 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14222 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14223 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14224 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
14225 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
14226 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14227 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
14228 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
14229 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14230 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14231 ; GFX12-WGP-NEXT: s_endpgm
14232 ;
14233 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
14234 ; GFX12-CU: ; %bb.0: ; %entry
14235 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14236 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
14237 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14238 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14239 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14240 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14241 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
14242 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
14243 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14244 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
14245 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14246 ; GFX12-CU-NEXT: s_endpgm
14247 ptr addrspace(1) %out, i32 %in, i32 %old) {
14248 entry:
14249 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
14250 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
14251 ret void
14252 }
14254 define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
14255 ; GFX6-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14256 ; GFX6: ; %bb.0: ; %entry
14257 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
14258 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
14259 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
14260 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
14261 ; GFX6-NEXT: s_mov_b32 s12, s5
14262 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
14263 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
14264 ; GFX6-NEXT: s_mov_b32 s11, -1
14265 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
14266 ; GFX6-NEXT: s_mov_b32 s5, s12
14267 ; GFX6-NEXT: s_mov_b32 s6, s11
14268 ; GFX6-NEXT: s_mov_b32 s7, s10
14269 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
14270 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
14271 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14272 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14273 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
14274 ; GFX6-NEXT: s_endpgm
14275 ;
14276 ; GFX7-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14277 ; GFX7: ; %bb.0: ; %entry
14278 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14279 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14280 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14281 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14282 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14283 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14284 ; GFX7-NEXT: s_mov_b32 s4, s8
14285 ; GFX7-NEXT: s_mov_b32 s5, s9
14286 ; GFX7-NEXT: s_mov_b32 s9, s10
14287 ; GFX7-NEXT: s_mov_b32 s8, s11
14288 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14289 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14290 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14291 ; GFX7-NEXT: s_mov_b32 s5, s8
14292 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14293 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14294 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14295 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14296 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14297 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14298 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14299 ; GFX7-NEXT: s_endpgm
14300 ;
14301 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14302 ; GFX10-WGP: ; %bb.0: ; %entry
14303 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
14304 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
14305 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14306 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
14307 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
14308 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14309 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
14310 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
14311 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14312 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
14313 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14314 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14315 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14316 ; GFX10-WGP-NEXT: s_endpgm
14317 ;
14318 ; GFX10-CU-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14319 ; GFX10-CU: ; %bb.0: ; %entry
14320 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
14321 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
14322 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14323 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
14324 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
14325 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14326 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
14327 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
14328 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14329 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
14330 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14331 ; GFX10-CU-NEXT: s_endpgm
14332 ;
14333 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14334 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14335 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
14336 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
14337 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
14338 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14339 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
14340 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
14341 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
14342 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
14343 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
14344 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
14345 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
14346 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
14347 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
14348 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
14349 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14350 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
14351 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
14352 ; SKIP-CACHE-INV-NEXT: s_endpgm
14353 ;
14354 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14355 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14356 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14357 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14358 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14359 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14360 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14361 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14362 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14363 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14364 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14365 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14366 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14367 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14368 ;
14369 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14370 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14371 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14372 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14373 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14374 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14375 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14376 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14377 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14378 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14379 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14380 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14381 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14382 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14383 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14384 ;
14385 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14386 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14387 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14388 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14389 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14390 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14391 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14392 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14393 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14394 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14395 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14396 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14397 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14398 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14399 ;
14400 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14401 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14402 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14403 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14404 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14405 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14406 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14407 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14408 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14409 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14410 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14411 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14412 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14413 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14414 ; GFX940-TGSPLIT-NEXT: s_endpgm
14415 ;
14416 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14417 ; GFX11-WGP: ; %bb.0: ; %entry
14418 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14419 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
14420 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14421 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14422 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14423 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14424 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
14425 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
14426 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14427 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
14428 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14429 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14430 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14431 ; GFX11-WGP-NEXT: s_endpgm
14432 ;
14433 ; GFX11-CU-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14434 ; GFX11-CU: ; %bb.0: ; %entry
14435 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14436 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
14437 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14438 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14439 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14440 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14441 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
14442 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
14443 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14444 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
14445 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14446 ; GFX11-CU-NEXT: s_endpgm
14447 ;
14448 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14449 ; GFX12-WGP: ; %bb.0: ; %entry
14450 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14451 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
14452 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14453 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14454 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14455 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14456 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
14457 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
14458 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14459 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
14460 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
14461 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14462 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14463 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14464 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14465 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
14466 ; GFX12-WGP-NEXT: s_endpgm
14467 ;
14468 ; GFX12-CU-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
14469 ; GFX12-CU: ; %bb.0: ; %entry
14470 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14471 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
14472 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14473 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14474 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14475 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14476 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
14477 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
14478 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14479 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
14480 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14481 ; GFX12-CU-NEXT: s_endpgm
14482 ptr addrspace(1) %out, i32 %in, i32 %old) {
14483 entry:
14484 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
14485 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
14486 ret void
14487 }
14489 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
14490 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14491 ; GFX6: ; %bb.0: ; %entry
14492 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
14493 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
14494 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
14495 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
14496 ; GFX6-NEXT: s_mov_b32 s12, s5
14497 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
14498 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
14499 ; GFX6-NEXT: s_mov_b32 s11, -1
14500 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
14501 ; GFX6-NEXT: s_mov_b32 s5, s12
14502 ; GFX6-NEXT: s_mov_b32 s6, s11
14503 ; GFX6-NEXT: s_mov_b32 s7, s10
14504 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
14505 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
14506 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14507 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14508 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
14509 ; GFX6-NEXT: s_endpgm
14510 ;
14511 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14512 ; GFX7: ; %bb.0: ; %entry
14513 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14514 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14515 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14516 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14517 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14518 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14519 ; GFX7-NEXT: s_mov_b32 s4, s8
14520 ; GFX7-NEXT: s_mov_b32 s5, s9
14521 ; GFX7-NEXT: s_mov_b32 s9, s10
14522 ; GFX7-NEXT: s_mov_b32 s8, s11
14523 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14524 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14525 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14526 ; GFX7-NEXT: s_mov_b32 s5, s8
14527 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14528 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14529 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14530 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14531 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14532 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14533 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14534 ; GFX7-NEXT: s_endpgm
14535 ;
14536 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14537 ; GFX10-WGP: ; %bb.0: ; %entry
14538 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
14539 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
14540 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14541 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
14542 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
14543 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14544 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
14545 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
14546 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14547 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
14548 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14549 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14550 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14551 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14552 ; GFX10-WGP-NEXT: buffer_gl0_inv
14553 ; GFX10-WGP-NEXT: s_endpgm
14554 ;
14555 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14556 ; GFX10-CU: ; %bb.0: ; %entry
14557 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
14558 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
14559 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14560 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
14561 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
14562 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14563 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
14564 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
14565 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14566 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
14567 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14568 ; GFX10-CU-NEXT: s_endpgm
14569 ;
14570 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14571 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14572 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
14573 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
14574 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
14575 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14576 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
14577 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
14578 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
14579 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
14580 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
14581 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
14582 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
14583 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
14584 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
14585 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
14586 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14587 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
14588 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
14589 ; SKIP-CACHE-INV-NEXT: s_endpgm
14590 ;
14591 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14592 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14593 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14594 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14595 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14596 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14597 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14598 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14599 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14600 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14601 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14602 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14603 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14604 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14605 ;
14606 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14607 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14608 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14609 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14610 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14611 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14612 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14613 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14614 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14615 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14616 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14617 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14618 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14619 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14620 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14621 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14622 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14623 ;
14624 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14625 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14626 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14627 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14628 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14629 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14630 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14631 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14632 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14633 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14634 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14635 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14636 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14637 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14638 ;
14639 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14640 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14641 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14642 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14643 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14644 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14645 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14646 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14647 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14648 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14649 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14650 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14651 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14652 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14653 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14654 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14655 ; GFX940-TGSPLIT-NEXT: s_endpgm
14656 ;
14657 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14658 ; GFX11-WGP: ; %bb.0: ; %entry
14659 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14660 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
14661 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14662 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14663 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14664 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14665 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
14666 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
14667 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14668 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
14669 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14670 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14671 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14672 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14673 ; GFX11-WGP-NEXT: buffer_gl0_inv
14674 ; GFX11-WGP-NEXT: s_endpgm
14675 ;
14676 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14677 ; GFX11-CU: ; %bb.0: ; %entry
14678 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14679 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
14680 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14681 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14682 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14683 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14684 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
14685 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
14686 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14687 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
14688 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14689 ; GFX11-CU-NEXT: s_endpgm
14690 ;
14691 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14692 ; GFX12-WGP: ; %bb.0: ; %entry
14693 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14694 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
14695 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14696 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14697 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14698 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14699 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
14700 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
14701 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14702 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
14703 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
14704 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14705 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14706 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14707 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14708 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
14709 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14710 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14711 ; GFX12-WGP-NEXT: s_endpgm
14712 ;
14713 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
14714 ; GFX12-CU: ; %bb.0: ; %entry
14715 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14716 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
14717 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14718 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14719 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14720 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14721 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
14722 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
14723 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14724 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
14725 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14726 ; GFX12-CU-NEXT: s_endpgm
14727 ptr addrspace(1) %out, i32 %in, i32 %old) {
14728 entry:
14729 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
14730 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
14731 ret void
14732 }
14734 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
14735 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14736 ; GFX6: ; %bb.0: ; %entry
14737 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
14738 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
14739 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
14740 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
14741 ; GFX6-NEXT: s_mov_b32 s12, s5
14742 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
14743 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
14744 ; GFX6-NEXT: s_mov_b32 s11, -1
14745 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
14746 ; GFX6-NEXT: s_mov_b32 s5, s12
14747 ; GFX6-NEXT: s_mov_b32 s6, s11
14748 ; GFX6-NEXT: s_mov_b32 s7, s10
14749 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
14750 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
14751 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14752 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14753 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
14754 ; GFX6-NEXT: s_endpgm
14755 ;
14756 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14757 ; GFX7: ; %bb.0: ; %entry
14758 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
14759 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
14760 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
14761 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
14762 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
14763 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14764 ; GFX7-NEXT: s_mov_b32 s4, s8
14765 ; GFX7-NEXT: s_mov_b32 s5, s9
14766 ; GFX7-NEXT: s_mov_b32 s9, s10
14767 ; GFX7-NEXT: s_mov_b32 s8, s11
14768 ; GFX7-NEXT: s_add_u32 s4, s4, s9
14769 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
14770 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
14771 ; GFX7-NEXT: s_mov_b32 s5, s8
14772 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
14773 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
14774 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14775 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
14776 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
14777 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
14778 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
14779 ; GFX7-NEXT: s_endpgm
14780 ;
14781 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14782 ; GFX10-WGP: ; %bb.0: ; %entry
14783 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
14784 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
14785 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14786 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
14787 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
14788 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
14789 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
14790 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
14791 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14792 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
14793 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
14794 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14795 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14796 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14797 ; GFX10-WGP-NEXT: buffer_gl0_inv
14798 ; GFX10-WGP-NEXT: s_endpgm
14799 ;
14800 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14801 ; GFX10-CU: ; %bb.0: ; %entry
14802 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
14803 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
14804 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14805 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
14806 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
14807 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
14808 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
14809 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
14810 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14811 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
14812 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
14813 ; GFX10-CU-NEXT: s_endpgm
14814 ;
14815 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14816 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
14817 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
14818 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
14819 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
14820 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
14821 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
14822 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
14823 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
14824 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
14825 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
14826 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
14827 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
14828 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
14829 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
14830 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
14831 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14832 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
14833 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
14834 ; SKIP-CACHE-INV-NEXT: s_endpgm
14835 ;
14836 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14837 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
14838 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14839 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14840 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14841 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14842 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14843 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14844 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14845 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14846 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14847 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14848 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14849 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
14850 ;
14851 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14852 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
14853 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
14854 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14855 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
14856 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
14857 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
14858 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14859 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
14860 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
14861 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14862 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14863 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14864 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
14865 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14866 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
14867 ; GFX90A-TGSPLIT-NEXT: s_endpgm
14868 ;
14869 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14870 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
14871 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14872 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14873 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14874 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14875 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14876 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14877 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14878 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14879 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14880 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14881 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14882 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
14883 ;
14884 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14885 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
14886 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
14887 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
14888 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
14889 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
14890 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
14891 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
14892 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
14893 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
14894 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
14895 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
14896 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14897 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
14898 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
14899 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
14900 ; GFX940-TGSPLIT-NEXT: s_endpgm
14901 ;
14902 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14903 ; GFX11-WGP: ; %bb.0: ; %entry
14904 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14905 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
14906 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14907 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14908 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14909 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
14910 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
14911 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
14912 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14913 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
14914 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
14915 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14916 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14917 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
14918 ; GFX11-WGP-NEXT: buffer_gl0_inv
14919 ; GFX11-WGP-NEXT: s_endpgm
14920 ;
14921 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14922 ; GFX11-CU: ; %bb.0: ; %entry
14923 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14924 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
14925 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14926 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14927 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14928 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
14929 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
14930 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
14931 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14932 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
14933 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14934 ; GFX11-CU-NEXT: s_endpgm
14935 ;
14936 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14937 ; GFX12-WGP: ; %bb.0: ; %entry
14938 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
14939 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
14940 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14941 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
14942 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
14943 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
14944 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
14945 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
14946 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14947 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
14948 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
14949 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
14950 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
14951 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
14952 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14953 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
14954 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
14955 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
14956 ; GFX12-WGP-NEXT: s_endpgm
14957 ;
14958 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
14959 ; GFX12-CU: ; %bb.0: ; %entry
14960 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
14961 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
14962 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
14963 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
14964 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
14965 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
14966 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
14967 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
14968 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
14969 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
14970 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
14971 ; GFX12-CU-NEXT: s_endpgm
14972 ptr addrspace(1) %out, i32 %in, i32 %old) {
14973 entry:
14974 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
14975 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
14976 ret void
14977 }
14979 define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_cmpxchg(
14980 ; GFX6-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
14981 ; GFX6: ; %bb.0: ; %entry
14982 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
14983 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
14984 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
14985 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
14986 ; GFX6-NEXT: s_mov_b32 s12, s5
14987 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
14988 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
14989 ; GFX6-NEXT: s_mov_b32 s11, -1
14990 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
14991 ; GFX6-NEXT: s_mov_b32 s5, s12
14992 ; GFX6-NEXT: s_mov_b32 s6, s11
14993 ; GFX6-NEXT: s_mov_b32 s7, s10
14994 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
14995 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
14996 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
14997 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
14998 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
14999 ; GFX6-NEXT: s_endpgm
15000 ;
15001 ; GFX7-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15002 ; GFX7: ; %bb.0: ; %entry
15003 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15004 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15005 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15006 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15007 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15008 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15009 ; GFX7-NEXT: s_mov_b32 s4, s8
15010 ; GFX7-NEXT: s_mov_b32 s5, s9
15011 ; GFX7-NEXT: s_mov_b32 s9, s10
15012 ; GFX7-NEXT: s_mov_b32 s8, s11
15013 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15014 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15015 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15016 ; GFX7-NEXT: s_mov_b32 s5, s8
15017 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15018 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15019 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15020 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15021 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15022 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15023 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15024 ; GFX7-NEXT: s_endpgm
15025 ;
15026 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15027 ; GFX10-WGP: ; %bb.0: ; %entry
15028 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
15029 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
15030 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15031 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
15032 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
15033 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15034 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
15035 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
15036 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15037 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
15038 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15039 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15040 ; GFX10-WGP-NEXT: buffer_gl0_inv
15041 ; GFX10-WGP-NEXT: s_endpgm
15042 ;
15043 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15044 ; GFX10-CU: ; %bb.0: ; %entry
15045 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
15046 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
15047 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15048 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
15049 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
15050 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15051 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
15052 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
15053 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15054 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
15055 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15056 ; GFX10-CU-NEXT: s_endpgm
15057 ;
15058 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15059 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15060 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
15061 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
15062 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
15063 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15064 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
15065 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
15066 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
15067 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
15068 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
15069 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
15070 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
15071 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
15072 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
15073 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
15074 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15075 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
15076 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
15077 ; SKIP-CACHE-INV-NEXT: s_endpgm
15078 ;
15079 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15080 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15081 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15082 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15083 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15084 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15085 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15086 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15087 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15088 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15089 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15090 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15091 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15092 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15093 ;
15094 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15095 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15096 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15097 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15098 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15099 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15100 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15101 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15102 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15103 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15104 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15105 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15106 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15107 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15108 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15109 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15110 ;
15111 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15112 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15113 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15114 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15115 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15116 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15117 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15118 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15119 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15120 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15121 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15122 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15123 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15124 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15125 ;
15126 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15127 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15128 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15129 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15130 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15131 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15132 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15133 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15134 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15135 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15136 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15137 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15138 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15139 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15140 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15141 ; GFX940-TGSPLIT-NEXT: s_endpgm
15142 ;
15143 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15144 ; GFX11-WGP: ; %bb.0: ; %entry
15145 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15146 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
15147 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15148 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15149 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15150 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15151 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
15152 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
15153 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15154 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
15155 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15156 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15157 ; GFX11-WGP-NEXT: buffer_gl0_inv
15158 ; GFX11-WGP-NEXT: s_endpgm
15159 ;
15160 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15161 ; GFX11-CU: ; %bb.0: ; %entry
15162 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15163 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
15164 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15165 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15166 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15167 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15168 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
15169 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
15170 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15171 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
15172 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15173 ; GFX11-CU-NEXT: s_endpgm
15174 ;
15175 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15176 ; GFX12-WGP: ; %bb.0: ; %entry
15177 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15178 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
15179 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15180 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15181 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15182 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15183 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
15184 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
15185 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15186 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
15187 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
15188 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15189 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15190 ; GFX12-WGP-NEXT: s_endpgm
15191 ;
15192 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
15193 ; GFX12-CU: ; %bb.0: ; %entry
15194 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15195 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
15196 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15197 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15198 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15199 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15200 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
15201 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
15202 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15203 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
15204 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15205 ; GFX12-CU-NEXT: s_endpgm
15206 ptr addrspace(1) %out, i32 %in, i32 %old) {
15207 entry:
15208 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
15209 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
15210 ret void
15211 }
15213 define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
15214 ; GFX6-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15215 ; GFX6: ; %bb.0: ; %entry
15216 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
15217 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
15218 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
15219 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
15220 ; GFX6-NEXT: s_mov_b32 s12, s5
15221 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
15222 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
15223 ; GFX6-NEXT: s_mov_b32 s11, -1
15224 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
15225 ; GFX6-NEXT: s_mov_b32 s5, s12
15226 ; GFX6-NEXT: s_mov_b32 s6, s11
15227 ; GFX6-NEXT: s_mov_b32 s7, s10
15228 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
15229 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
15230 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15231 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15232 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
15233 ; GFX6-NEXT: s_endpgm
15234 ;
15235 ; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15236 ; GFX7: ; %bb.0: ; %entry
15237 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15238 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15239 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15240 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15241 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15242 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15243 ; GFX7-NEXT: s_mov_b32 s4, s8
15244 ; GFX7-NEXT: s_mov_b32 s5, s9
15245 ; GFX7-NEXT: s_mov_b32 s9, s10
15246 ; GFX7-NEXT: s_mov_b32 s8, s11
15247 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15248 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15249 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15250 ; GFX7-NEXT: s_mov_b32 s5, s8
15251 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15252 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15253 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15254 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15255 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15256 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15257 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15258 ; GFX7-NEXT: s_endpgm
15259 ;
15260 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15261 ; GFX10-WGP: ; %bb.0: ; %entry
15262 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
15263 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
15264 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15265 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
15266 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
15267 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15268 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
15269 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
15270 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15271 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
15272 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15273 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15274 ; GFX10-WGP-NEXT: buffer_gl0_inv
15275 ; GFX10-WGP-NEXT: s_endpgm
15276 ;
15277 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15278 ; GFX10-CU: ; %bb.0: ; %entry
15279 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
15280 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
15281 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15282 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
15283 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
15284 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15285 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
15286 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
15287 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15288 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
15289 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15290 ; GFX10-CU-NEXT: s_endpgm
15291 ;
15292 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15293 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15294 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
15295 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
15296 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
15297 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15298 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
15299 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
15300 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
15301 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
15302 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
15303 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
15304 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
15305 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
15306 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
15307 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
15308 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15309 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
15310 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
15311 ; SKIP-CACHE-INV-NEXT: s_endpgm
15312 ;
15313 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15314 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15315 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15316 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15317 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15318 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15319 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15320 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15321 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15322 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15323 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15324 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15325 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15326 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15327 ;
15328 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15329 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15330 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15331 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15332 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15333 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15334 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15335 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15336 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15337 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15338 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15339 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15340 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15341 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15342 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15343 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15344 ;
15345 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15346 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15347 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15348 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15349 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15350 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15351 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15352 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15353 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15354 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15355 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15356 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15357 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15358 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15359 ;
15360 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15361 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15362 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15363 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15364 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15365 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15366 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15367 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15368 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15369 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15370 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15371 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15372 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15373 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15374 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15375 ; GFX940-TGSPLIT-NEXT: s_endpgm
15376 ;
15377 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15378 ; GFX11-WGP: ; %bb.0: ; %entry
15379 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15380 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
15381 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15382 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15383 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15384 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15385 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
15386 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
15387 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15388 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
15389 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15390 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15391 ; GFX11-WGP-NEXT: buffer_gl0_inv
15392 ; GFX11-WGP-NEXT: s_endpgm
15393 ;
15394 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15395 ; GFX11-CU: ; %bb.0: ; %entry
15396 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15397 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
15398 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15399 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15400 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15401 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15402 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
15403 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
15404 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15405 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
15406 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15407 ; GFX11-CU-NEXT: s_endpgm
15408 ;
15409 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15410 ; GFX12-WGP: ; %bb.0: ; %entry
15411 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15412 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
15413 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15414 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15415 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15416 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15417 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
15418 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
15419 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15420 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
15421 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
15422 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15423 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15424 ; GFX12-WGP-NEXT: s_endpgm
15425 ;
15426 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
15427 ; GFX12-CU: ; %bb.0: ; %entry
15428 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15429 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
15430 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15431 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15432 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15433 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15434 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
15435 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
15436 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15437 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
15438 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15439 ; GFX12-CU-NEXT: s_endpgm
15440 ptr addrspace(1) %out, i32 %in, i32 %old) {
15441 entry:
15442 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
15443 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
15444 ret void
15445 }
15447 define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
15448 ; GFX6-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15449 ; GFX6: ; %bb.0: ; %entry
15450 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
15451 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
15452 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
15453 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
15454 ; GFX6-NEXT: s_mov_b32 s12, s5
15455 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
15456 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
15457 ; GFX6-NEXT: s_mov_b32 s11, -1
15458 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
15459 ; GFX6-NEXT: s_mov_b32 s5, s12
15460 ; GFX6-NEXT: s_mov_b32 s6, s11
15461 ; GFX6-NEXT: s_mov_b32 s7, s10
15462 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
15463 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
15464 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15465 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15466 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
15467 ; GFX6-NEXT: s_endpgm
15468 ;
15469 ; GFX7-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15470 ; GFX7: ; %bb.0: ; %entry
15471 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15472 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15473 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15474 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15475 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15476 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15477 ; GFX7-NEXT: s_mov_b32 s4, s8
15478 ; GFX7-NEXT: s_mov_b32 s5, s9
15479 ; GFX7-NEXT: s_mov_b32 s9, s10
15480 ; GFX7-NEXT: s_mov_b32 s8, s11
15481 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15482 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15483 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15484 ; GFX7-NEXT: s_mov_b32 s5, s8
15485 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15486 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15487 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15488 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15489 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15490 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15491 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15492 ; GFX7-NEXT: s_endpgm
15493 ;
15494 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15495 ; GFX10-WGP: ; %bb.0: ; %entry
15496 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
15497 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
15498 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15499 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
15500 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
15501 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15502 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
15503 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
15504 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15505 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
15506 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15507 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15508 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15509 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15510 ; GFX10-WGP-NEXT: buffer_gl0_inv
15511 ; GFX10-WGP-NEXT: s_endpgm
15512 ;
15513 ; GFX10-CU-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15514 ; GFX10-CU: ; %bb.0: ; %entry
15515 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
15516 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
15517 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15518 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
15519 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
15520 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15521 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
15522 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
15523 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15524 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
15525 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15526 ; GFX10-CU-NEXT: s_endpgm
15527 ;
15528 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15529 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15530 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
15531 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
15532 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
15533 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15534 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
15535 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
15536 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
15537 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
15538 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
15539 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
15540 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
15541 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
15542 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
15543 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
15544 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15545 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
15546 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
15547 ; SKIP-CACHE-INV-NEXT: s_endpgm
15548 ;
15549 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15550 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15551 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15552 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15553 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15554 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15555 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15556 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15557 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15558 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15559 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15560 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15561 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15562 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15563 ;
15564 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15565 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15566 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15567 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15568 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15569 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15570 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15571 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15572 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15573 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15574 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15575 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15576 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15577 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15578 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15579 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15580 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15581 ;
15582 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15583 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15584 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15585 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15586 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15587 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15588 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15589 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15590 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15591 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15592 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15593 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15594 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15595 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15596 ;
15597 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15598 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15599 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15600 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15601 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15602 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15603 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15604 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15605 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15606 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15607 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15608 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15609 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15610 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15611 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15612 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15613 ; GFX940-TGSPLIT-NEXT: s_endpgm
15614 ;
15615 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15616 ; GFX11-WGP: ; %bb.0: ; %entry
15617 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15618 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
15619 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15620 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15621 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15622 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15623 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
15624 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
15625 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15626 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
15627 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15628 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15629 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15630 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15631 ; GFX11-WGP-NEXT: buffer_gl0_inv
15632 ; GFX11-WGP-NEXT: s_endpgm
15633 ;
15634 ; GFX11-CU-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15635 ; GFX11-CU: ; %bb.0: ; %entry
15636 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15637 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
15638 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15639 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15640 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15641 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15642 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
15643 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
15644 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15645 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
15646 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15647 ; GFX11-CU-NEXT: s_endpgm
15648 ;
15649 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15650 ; GFX12-WGP: ; %bb.0: ; %entry
15651 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15652 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
15653 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15654 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15655 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15656 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15657 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
15658 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
15659 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15660 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
15661 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15662 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15663 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15664 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15665 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15666 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
15667 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15668 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15669 ; GFX12-WGP-NEXT: s_endpgm
15670 ;
15671 ; GFX12-CU-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
15672 ; GFX12-CU: ; %bb.0: ; %entry
15673 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15674 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
15675 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15676 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15677 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15678 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15679 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
15680 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
15681 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15682 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
15683 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15684 ; GFX12-CU-NEXT: s_endpgm
15685 ptr addrspace(1) %out, i32 %in, i32 %old) {
15686 entry:
15687 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
15688 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
15689 ret void
15690 }
15692 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
15693 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15694 ; GFX6: ; %bb.0: ; %entry
15695 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
15696 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
15697 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
15698 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
15699 ; GFX6-NEXT: s_mov_b32 s12, s5
15700 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
15701 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
15702 ; GFX6-NEXT: s_mov_b32 s11, -1
15703 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
15704 ; GFX6-NEXT: s_mov_b32 s5, s12
15705 ; GFX6-NEXT: s_mov_b32 s6, s11
15706 ; GFX6-NEXT: s_mov_b32 s7, s10
15707 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
15708 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
15709 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15710 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15711 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
15712 ; GFX6-NEXT: s_endpgm
15713 ;
15714 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15715 ; GFX7: ; %bb.0: ; %entry
15716 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15717 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15718 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15719 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15720 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15721 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15722 ; GFX7-NEXT: s_mov_b32 s4, s8
15723 ; GFX7-NEXT: s_mov_b32 s5, s9
15724 ; GFX7-NEXT: s_mov_b32 s9, s10
15725 ; GFX7-NEXT: s_mov_b32 s8, s11
15726 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15727 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15728 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15729 ; GFX7-NEXT: s_mov_b32 s5, s8
15730 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15731 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15732 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15733 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15734 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15735 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15736 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15737 ; GFX7-NEXT: s_endpgm
15738 ;
15739 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15740 ; GFX10-WGP: ; %bb.0: ; %entry
15741 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
15742 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
15743 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15744 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
15745 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
15746 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15747 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
15748 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
15749 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15750 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
15751 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15752 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15753 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15754 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15755 ; GFX10-WGP-NEXT: buffer_gl0_inv
15756 ; GFX10-WGP-NEXT: s_endpgm
15757 ;
15758 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15759 ; GFX10-CU: ; %bb.0: ; %entry
15760 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
15761 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
15762 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15763 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
15764 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
15765 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
15766 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
15767 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
15768 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15769 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
15770 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15771 ; GFX10-CU-NEXT: s_endpgm
15772 ;
15773 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15774 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
15775 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
15776 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
15777 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
15778 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
15779 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
15780 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
15781 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
15782 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
15783 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
15784 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
15785 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
15786 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
15787 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
15788 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
15789 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15790 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
15791 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
15792 ; SKIP-CACHE-INV-NEXT: s_endpgm
15793 ;
15794 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15795 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
15796 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15797 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15798 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15799 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15800 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15801 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15802 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15803 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15804 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15805 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15806 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15807 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
15808 ;
15809 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15810 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
15811 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
15812 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15813 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15814 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
15815 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
15816 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15817 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
15818 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
15819 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15820 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15821 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15822 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
15823 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15824 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
15825 ; GFX90A-TGSPLIT-NEXT: s_endpgm
15826 ;
15827 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15828 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
15829 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15830 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15831 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15832 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15833 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15834 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15835 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15836 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15837 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15838 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15839 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15840 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
15841 ;
15842 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15843 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
15844 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
15845 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
15846 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
15847 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
15848 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
15849 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
15850 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
15851 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
15852 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15853 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
15854 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15855 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
15856 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
15857 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
15858 ; GFX940-TGSPLIT-NEXT: s_endpgm
15859 ;
15860 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15861 ; GFX11-WGP: ; %bb.0: ; %entry
15862 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15863 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
15864 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15865 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15866 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15867 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
15868 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
15869 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
15870 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15871 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
15872 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
15873 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15874 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15875 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15876 ; GFX11-WGP-NEXT: buffer_gl0_inv
15877 ; GFX11-WGP-NEXT: s_endpgm
15878 ;
15879 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15880 ; GFX11-CU: ; %bb.0: ; %entry
15881 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15882 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
15883 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15884 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15885 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15886 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
15887 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
15888 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
15889 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15890 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
15891 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15892 ; GFX11-CU-NEXT: s_endpgm
15893 ;
15894 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15895 ; GFX12-WGP: ; %bb.0: ; %entry
15896 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
15897 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
15898 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15899 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
15900 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
15901 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
15902 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
15903 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
15904 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15905 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
15906 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
15907 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
15908 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
15909 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
15910 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15911 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
15912 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
15913 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
15914 ; GFX12-WGP-NEXT: s_endpgm
15915 ;
15916 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
15917 ; GFX12-CU: ; %bb.0: ; %entry
15918 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
15919 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
15920 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
15921 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
15922 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
15923 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
15924 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
15925 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
15926 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15927 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
15928 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
15929 ; GFX12-CU-NEXT: s_endpgm
15930 ptr addrspace(1) %out, i32 %in, i32 %old) {
15931 entry:
15932 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
15933 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
15934 ret void
15935 }
15937 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
15938 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
15939 ; GFX6: ; %bb.0: ; %entry
15940 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
15941 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
15942 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
15943 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
15944 ; GFX6-NEXT: s_mov_b32 s12, s5
15945 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
15946 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
15947 ; GFX6-NEXT: s_mov_b32 s11, -1
15948 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
15949 ; GFX6-NEXT: s_mov_b32 s5, s12
15950 ; GFX6-NEXT: s_mov_b32 s6, s11
15951 ; GFX6-NEXT: s_mov_b32 s7, s10
15952 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
15953 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
15954 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
15955 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
15956 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
15957 ; GFX6-NEXT: s_endpgm
15958 ;
15959 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
15960 ; GFX7: ; %bb.0: ; %entry
15961 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
15962 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
15963 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
15964 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
15965 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
15966 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
15967 ; GFX7-NEXT: s_mov_b32 s4, s8
15968 ; GFX7-NEXT: s_mov_b32 s5, s9
15969 ; GFX7-NEXT: s_mov_b32 s9, s10
15970 ; GFX7-NEXT: s_mov_b32 s8, s11
15971 ; GFX7-NEXT: s_add_u32 s4, s4, s9
15972 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
15973 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
15974 ; GFX7-NEXT: s_mov_b32 s5, s8
15975 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
15976 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
15977 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
15978 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
15979 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
15980 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
15981 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
15982 ; GFX7-NEXT: s_endpgm
15983 ;
15984 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
15985 ; GFX10-WGP: ; %bb.0: ; %entry
15986 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
15987 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
15988 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
15989 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
15990 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
15991 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
15992 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
15993 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
15994 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
15995 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
15996 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
15997 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
15998 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
15999 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16000 ; GFX10-WGP-NEXT: buffer_gl0_inv
16001 ; GFX10-WGP-NEXT: s_endpgm
16002 ;
16003 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16004 ; GFX10-CU: ; %bb.0: ; %entry
16005 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
16006 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
16007 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16008 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
16009 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
16010 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16011 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16012 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
16013 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16014 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
16015 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16016 ; GFX10-CU-NEXT: s_endpgm
16017 ;
16018 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16019 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16020 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
16021 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
16022 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
16023 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16024 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
16025 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
16026 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
16027 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
16028 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
16029 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
16030 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
16031 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
16032 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
16033 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
16034 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16035 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
16036 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
16037 ; SKIP-CACHE-INV-NEXT: s_endpgm
16038 ;
16039 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16040 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16041 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16042 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16043 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16044 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16045 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16046 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16047 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16048 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16049 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16050 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16051 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16052 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16053 ;
16054 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16055 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16056 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16057 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16058 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16059 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16060 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16061 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16062 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16063 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16064 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16065 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16066 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16067 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16068 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16069 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16070 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16071 ;
16072 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16073 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16074 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16075 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16076 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16077 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16078 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16079 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16080 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16081 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16082 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16083 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16084 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16085 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16086 ;
16087 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16088 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16089 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16090 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16091 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16092 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16093 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16094 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16095 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16096 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16097 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16098 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16099 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16100 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16101 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16102 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16103 ; GFX940-TGSPLIT-NEXT: s_endpgm
16104 ;
16105 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16106 ; GFX11-WGP: ; %bb.0: ; %entry
16107 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16108 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
16109 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16110 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16111 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16112 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16113 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
16114 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
16115 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16116 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
16117 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16118 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16119 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16120 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16121 ; GFX11-WGP-NEXT: buffer_gl0_inv
16122 ; GFX11-WGP-NEXT: s_endpgm
16123 ;
16124 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16125 ; GFX11-CU: ; %bb.0: ; %entry
16126 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16127 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
16128 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16129 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16130 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16131 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16132 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
16133 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
16134 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16135 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
16136 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16137 ; GFX11-CU-NEXT: s_endpgm
16138 ;
16139 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16140 ; GFX12-WGP: ; %bb.0: ; %entry
16141 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16142 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
16143 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16144 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16145 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16146 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16147 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
16148 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
16149 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16150 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
16151 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16152 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16153 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16154 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16155 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16156 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
16157 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16158 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16159 ; GFX12-WGP-NEXT: s_endpgm
16160 ;
16161 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
16162 ; GFX12-CU: ; %bb.0: ; %entry
16163 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16164 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
16165 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16166 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16167 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16168 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16169 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
16170 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
16171 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16172 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
16173 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16174 ; GFX12-CU-NEXT: s_endpgm
16175 ptr addrspace(1) %out, i32 %in, i32 %old) {
16176 entry:
16177 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
16178 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
16179 ret void
16180 }
16182 define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_cmpxchg(
16183 ; GFX6-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16184 ; GFX6: ; %bb.0: ; %entry
16185 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16186 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
16187 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
16188 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
16189 ; GFX6-NEXT: s_mov_b32 s12, s5
16190 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
16191 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
16192 ; GFX6-NEXT: s_mov_b32 s11, -1
16193 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
16194 ; GFX6-NEXT: s_mov_b32 s5, s12
16195 ; GFX6-NEXT: s_mov_b32 s6, s11
16196 ; GFX6-NEXT: s_mov_b32 s7, s10
16197 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
16198 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
16199 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16200 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
16201 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
16202 ; GFX6-NEXT: s_endpgm
16203 ;
16204 ; GFX7-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16205 ; GFX7: ; %bb.0: ; %entry
16206 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16207 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16208 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16209 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16210 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16211 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16212 ; GFX7-NEXT: s_mov_b32 s4, s8
16213 ; GFX7-NEXT: s_mov_b32 s5, s9
16214 ; GFX7-NEXT: s_mov_b32 s9, s10
16215 ; GFX7-NEXT: s_mov_b32 s8, s11
16216 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16217 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16218 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16219 ; GFX7-NEXT: s_mov_b32 s5, s8
16220 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16221 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16222 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16223 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16224 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16225 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16226 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16227 ; GFX7-NEXT: s_endpgm
16228 ;
16229 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16230 ; GFX10-WGP: ; %bb.0: ; %entry
16231 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
16232 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
16233 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16234 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
16235 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
16236 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16237 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
16238 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
16239 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16240 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
16241 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16242 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16243 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16244 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16245 ; GFX10-WGP-NEXT: buffer_gl0_inv
16246 ; GFX10-WGP-NEXT: s_endpgm
16247 ;
16248 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16249 ; GFX10-CU: ; %bb.0: ; %entry
16250 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
16251 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
16252 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16253 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
16254 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
16255 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16256 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16257 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
16258 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16259 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
16260 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16261 ; GFX10-CU-NEXT: s_endpgm
16262 ;
16263 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16264 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16265 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
16266 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
16267 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
16268 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16269 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
16270 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
16271 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
16272 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
16273 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
16274 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
16275 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
16276 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
16277 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
16278 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
16279 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16280 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
16281 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
16282 ; SKIP-CACHE-INV-NEXT: s_endpgm
16283 ;
16284 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16285 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16286 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16287 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16288 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16289 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16290 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16291 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16292 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16293 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16294 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16295 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16296 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16297 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16298 ;
16299 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16300 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16301 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16302 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16303 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16304 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16305 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16306 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16307 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16308 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16309 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16310 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16311 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16312 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16313 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16314 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16315 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16316 ;
16317 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16318 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16319 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16320 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16321 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16322 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16323 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16324 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16325 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16326 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16327 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16328 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16329 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16330 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16331 ;
16332 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16333 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16334 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16335 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16336 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16337 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16338 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16339 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16340 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16341 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16342 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16343 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16344 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16345 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16346 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16347 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16348 ; GFX940-TGSPLIT-NEXT: s_endpgm
16349 ;
16350 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16351 ; GFX11-WGP: ; %bb.0: ; %entry
16352 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16353 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
16354 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16355 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16356 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16357 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16358 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
16359 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
16360 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16361 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
16362 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16363 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16364 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16365 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16366 ; GFX11-WGP-NEXT: buffer_gl0_inv
16367 ; GFX11-WGP-NEXT: s_endpgm
16368 ;
16369 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16370 ; GFX11-CU: ; %bb.0: ; %entry
16371 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16372 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
16373 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16374 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16375 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16376 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16377 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
16378 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
16379 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16380 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
16381 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16382 ; GFX11-CU-NEXT: s_endpgm
16383 ;
16384 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16385 ; GFX12-WGP: ; %bb.0: ; %entry
16386 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16387 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
16388 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16389 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16390 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16391 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16392 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
16393 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
16394 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16395 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
16396 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16397 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16398 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16399 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16400 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16401 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
16402 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16403 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16404 ; GFX12-WGP-NEXT: s_endpgm
16405 ;
16406 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
16407 ; GFX12-CU: ; %bb.0: ; %entry
16408 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16409 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
16410 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16411 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16412 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16413 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16414 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
16415 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
16416 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16417 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
16418 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16419 ; GFX12-CU-NEXT: s_endpgm
16420 ptr addrspace(1) %out, i32 %in, i32 %old) {
16421 entry:
16422 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
16423 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
16424 ret void
16425 }
16427 define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_cmpxchg(
16428 ; GFX6-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16429 ; GFX6: ; %bb.0: ; %entry
16430 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16431 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
16432 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
16433 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
16434 ; GFX6-NEXT: s_mov_b32 s12, s5
16435 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
16436 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
16437 ; GFX6-NEXT: s_mov_b32 s11, -1
16438 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
16439 ; GFX6-NEXT: s_mov_b32 s5, s12
16440 ; GFX6-NEXT: s_mov_b32 s6, s11
16441 ; GFX6-NEXT: s_mov_b32 s7, s10
16442 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
16443 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
16444 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16445 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
16446 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
16447 ; GFX6-NEXT: s_endpgm
16448 ;
16449 ; GFX7-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16450 ; GFX7: ; %bb.0: ; %entry
16451 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16452 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16453 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16454 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16455 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16456 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16457 ; GFX7-NEXT: s_mov_b32 s4, s8
16458 ; GFX7-NEXT: s_mov_b32 s5, s9
16459 ; GFX7-NEXT: s_mov_b32 s9, s10
16460 ; GFX7-NEXT: s_mov_b32 s8, s11
16461 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16462 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16463 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16464 ; GFX7-NEXT: s_mov_b32 s5, s8
16465 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16466 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16467 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16468 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16469 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16470 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16471 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16472 ; GFX7-NEXT: s_endpgm
16473 ;
16474 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16475 ; GFX10-WGP: ; %bb.0: ; %entry
16476 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
16477 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
16478 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16479 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
16480 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
16481 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16482 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
16483 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
16484 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16485 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
16486 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16487 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16488 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16489 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16490 ; GFX10-WGP-NEXT: buffer_gl0_inv
16491 ; GFX10-WGP-NEXT: s_endpgm
16492 ;
16493 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16494 ; GFX10-CU: ; %bb.0: ; %entry
16495 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
16496 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
16497 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16498 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
16499 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
16500 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16501 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16502 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
16503 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16504 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
16505 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16506 ; GFX10-CU-NEXT: s_endpgm
16507 ;
16508 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16509 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16510 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
16511 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
16512 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
16513 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16514 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
16515 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
16516 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
16517 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
16518 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
16519 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
16520 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
16521 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
16522 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
16523 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
16524 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16525 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
16526 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
16527 ; SKIP-CACHE-INV-NEXT: s_endpgm
16528 ;
16529 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16530 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16531 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16532 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16533 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16534 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16535 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16536 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16537 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16538 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16539 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16540 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16541 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16542 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16543 ;
16544 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16545 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16546 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16547 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16548 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16549 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16550 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16551 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16552 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16553 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16554 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16555 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16556 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16557 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16558 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16559 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16560 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16561 ;
16562 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16563 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16564 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16565 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16566 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16567 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16568 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16569 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16570 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16571 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16572 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16573 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16574 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16575 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16576 ;
16577 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16578 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16579 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16580 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16581 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16582 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16583 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16584 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16585 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16586 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16587 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16588 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16589 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16590 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16591 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16592 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16593 ; GFX940-TGSPLIT-NEXT: s_endpgm
16594 ;
16595 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16596 ; GFX11-WGP: ; %bb.0: ; %entry
16597 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16598 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
16599 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16600 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16601 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16602 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16603 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
16604 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
16605 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16606 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
16607 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16608 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16609 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16610 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16611 ; GFX11-WGP-NEXT: buffer_gl0_inv
16612 ; GFX11-WGP-NEXT: s_endpgm
16613 ;
16614 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16615 ; GFX11-CU: ; %bb.0: ; %entry
16616 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16617 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
16618 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16619 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16620 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16621 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16622 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
16623 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
16624 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16625 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
16626 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16627 ; GFX11-CU-NEXT: s_endpgm
16628 ;
16629 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16630 ; GFX12-WGP: ; %bb.0: ; %entry
16631 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16632 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
16633 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16634 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16635 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16636 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16637 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
16638 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
16639 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16640 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
16641 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16642 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16643 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16644 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16645 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16646 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
16647 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16648 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16649 ; GFX12-WGP-NEXT: s_endpgm
16650 ;
16651 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
16652 ; GFX12-CU: ; %bb.0: ; %entry
16653 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16654 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
16655 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16656 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16657 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16658 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16659 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
16660 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
16661 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16662 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
16663 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16664 ; GFX12-CU-NEXT: s_endpgm
16665 ptr addrspace(1) %out, i32 %in, i32 %old) {
16666 entry:
16667 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
16668 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
16669 ret void
16670 }
16672 define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_cmpxchg(
16673 ; GFX6-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16674 ; GFX6: ; %bb.0: ; %entry
16675 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16676 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
16677 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
16678 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
16679 ; GFX6-NEXT: s_mov_b32 s12, s5
16680 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
16681 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
16682 ; GFX6-NEXT: s_mov_b32 s11, -1
16683 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
16684 ; GFX6-NEXT: s_mov_b32 s5, s12
16685 ; GFX6-NEXT: s_mov_b32 s6, s11
16686 ; GFX6-NEXT: s_mov_b32 s7, s10
16687 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
16688 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
16689 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16690 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
16691 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
16692 ; GFX6-NEXT: s_endpgm
16693 ;
16694 ; GFX7-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16695 ; GFX7: ; %bb.0: ; %entry
16696 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16697 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16698 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16699 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16700 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16701 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16702 ; GFX7-NEXT: s_mov_b32 s4, s8
16703 ; GFX7-NEXT: s_mov_b32 s5, s9
16704 ; GFX7-NEXT: s_mov_b32 s9, s10
16705 ; GFX7-NEXT: s_mov_b32 s8, s11
16706 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16707 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16708 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16709 ; GFX7-NEXT: s_mov_b32 s5, s8
16710 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16711 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16712 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16713 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16714 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16715 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16716 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16717 ; GFX7-NEXT: s_endpgm
16718 ;
16719 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16720 ; GFX10-WGP: ; %bb.0: ; %entry
16721 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
16722 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
16723 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16724 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
16725 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
16726 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16727 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
16728 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
16729 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16730 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
16731 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16732 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16733 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16734 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16735 ; GFX10-WGP-NEXT: buffer_gl0_inv
16736 ; GFX10-WGP-NEXT: s_endpgm
16737 ;
16738 ; GFX10-CU-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16739 ; GFX10-CU: ; %bb.0: ; %entry
16740 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
16741 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
16742 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16743 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
16744 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
16745 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16746 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16747 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
16748 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16749 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
16750 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16751 ; GFX10-CU-NEXT: s_endpgm
16752 ;
16753 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16754 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
16755 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
16756 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
16757 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
16758 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
16759 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
16760 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
16761 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
16762 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
16763 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
16764 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
16765 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
16766 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
16767 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
16768 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
16769 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16770 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
16771 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
16772 ; SKIP-CACHE-INV-NEXT: s_endpgm
16773 ;
16774 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16775 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
16776 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16777 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16778 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16779 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16780 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16781 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16782 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16783 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16784 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16785 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16786 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16787 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
16788 ;
16789 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16790 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
16791 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
16792 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16793 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16794 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
16795 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
16796 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16797 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
16798 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
16799 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16800 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16801 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16802 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
16803 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16804 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
16805 ; GFX90A-TGSPLIT-NEXT: s_endpgm
16806 ;
16807 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16808 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
16809 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16810 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16811 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16812 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16813 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16814 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16815 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16816 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16817 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16818 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16819 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16820 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
16821 ;
16822 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16823 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
16824 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
16825 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
16826 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
16827 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
16828 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
16829 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
16830 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
16831 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
16832 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16833 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
16834 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16835 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
16836 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
16837 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
16838 ; GFX940-TGSPLIT-NEXT: s_endpgm
16839 ;
16840 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16841 ; GFX11-WGP: ; %bb.0: ; %entry
16842 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16843 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
16844 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16845 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16846 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16847 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
16848 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
16849 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
16850 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16851 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
16852 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
16853 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16854 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16855 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16856 ; GFX11-WGP-NEXT: buffer_gl0_inv
16857 ; GFX11-WGP-NEXT: s_endpgm
16858 ;
16859 ; GFX11-CU-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16860 ; GFX11-CU: ; %bb.0: ; %entry
16861 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16862 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
16863 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16864 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16865 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16866 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
16867 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
16868 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
16869 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16870 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
16871 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16872 ; GFX11-CU-NEXT: s_endpgm
16873 ;
16874 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16875 ; GFX12-WGP: ; %bb.0: ; %entry
16876 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
16877 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
16878 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16879 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
16880 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
16881 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
16882 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
16883 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
16884 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16885 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
16886 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
16887 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
16888 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
16889 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
16890 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16891 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
16892 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
16893 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
16894 ; GFX12-WGP-NEXT: s_endpgm
16895 ;
16896 ; GFX12-CU-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
16897 ; GFX12-CU: ; %bb.0: ; %entry
16898 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
16899 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
16900 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
16901 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
16902 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
16903 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
16904 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
16905 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
16906 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16907 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
16908 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
16909 ; GFX12-CU-NEXT: s_endpgm
16910 ptr addrspace(1) %out, i32 %in, i32 %old) {
16911 entry:
16912 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
16913 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
16914 ret void
16915 }
16917 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
16918 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16919 ; GFX6: ; %bb.0: ; %entry
16920 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
16921 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
16922 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
16923 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
16924 ; GFX6-NEXT: s_mov_b32 s12, s5
16925 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
16926 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
16927 ; GFX6-NEXT: s_mov_b32 s11, -1
16928 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
16929 ; GFX6-NEXT: s_mov_b32 s5, s12
16930 ; GFX6-NEXT: s_mov_b32 s6, s11
16931 ; GFX6-NEXT: s_mov_b32 s7, s10
16932 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
16933 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
16934 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
16935 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
16936 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
16937 ; GFX6-NEXT: s_endpgm
16938 ;
16939 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16940 ; GFX7: ; %bb.0: ; %entry
16941 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
16942 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
16943 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
16944 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
16945 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
16946 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
16947 ; GFX7-NEXT: s_mov_b32 s4, s8
16948 ; GFX7-NEXT: s_mov_b32 s5, s9
16949 ; GFX7-NEXT: s_mov_b32 s9, s10
16950 ; GFX7-NEXT: s_mov_b32 s8, s11
16951 ; GFX7-NEXT: s_add_u32 s4, s4, s9
16952 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
16953 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
16954 ; GFX7-NEXT: s_mov_b32 s5, s8
16955 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
16956 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
16957 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
16958 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
16959 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
16960 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
16961 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
16962 ; GFX7-NEXT: s_endpgm
16963 ;
16964 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16965 ; GFX10-WGP: ; %bb.0: ; %entry
16966 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
16967 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
16968 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16969 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
16970 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
16971 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
16972 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
16973 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
16974 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16975 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
16976 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
16977 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16978 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16979 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
16980 ; GFX10-WGP-NEXT: buffer_gl0_inv
16981 ; GFX10-WGP-NEXT: s_endpgm
16982 ;
16983 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16984 ; GFX10-CU: ; %bb.0: ; %entry
16985 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
16986 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
16987 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
16988 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
16989 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
16990 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
16991 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
16992 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
16993 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
16994 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
16995 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
16996 ; GFX10-CU-NEXT: s_endpgm
16997 ;
16998 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
16999 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17000 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17001 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17002 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17003 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17004 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
17005 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
17006 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
17007 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
17008 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
17009 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
17010 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
17011 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17012 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
17013 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
17014 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17015 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
17016 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
17017 ; SKIP-CACHE-INV-NEXT: s_endpgm
17018 ;
17019 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17020 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17021 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17022 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17023 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17024 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17025 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17026 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17027 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17028 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17029 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17030 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17031 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
17032 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17033 ;
17034 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17035 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17036 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17037 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17038 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17039 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17040 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17041 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17042 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17043 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17044 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17045 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17046 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17047 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
17048 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17049 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17050 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17051 ;
17052 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17053 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17054 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17055 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17056 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17057 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17058 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17059 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17060 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17061 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17062 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17063 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17064 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
17065 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17066 ;
17067 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17068 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17069 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17070 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17071 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17072 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17073 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17074 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17075 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17076 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17077 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17078 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17079 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17080 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
17081 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17082 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
17083 ; GFX940-TGSPLIT-NEXT: s_endpgm
17084 ;
17085 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17086 ; GFX11-WGP: ; %bb.0: ; %entry
17087 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17088 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
17089 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17090 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17091 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17092 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17093 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
17094 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
17095 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17096 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
17097 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17098 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17099 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17100 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17101 ; GFX11-WGP-NEXT: buffer_gl0_inv
17102 ; GFX11-WGP-NEXT: s_endpgm
17103 ;
17104 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17105 ; GFX11-CU: ; %bb.0: ; %entry
17106 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17107 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
17108 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17109 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17110 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17111 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17112 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
17113 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
17114 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17115 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
17116 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17117 ; GFX11-CU-NEXT: s_endpgm
17118 ;
17119 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17120 ; GFX12-WGP: ; %bb.0: ; %entry
17121 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17122 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
17123 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17124 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17125 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17126 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17127 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
17128 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
17129 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17130 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
17131 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
17132 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17133 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17134 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17135 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17136 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
17137 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17138 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
17139 ; GFX12-WGP-NEXT: s_endpgm
17140 ;
17141 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
17142 ; GFX12-CU: ; %bb.0: ; %entry
17143 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17144 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
17145 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17146 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17147 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17148 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17149 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
17150 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
17151 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17152 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
17153 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17154 ; GFX12-CU-NEXT: s_endpgm
17155 ptr addrspace(1) %out, i32 %in, i32 %old) {
17156 entry:
17157 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
17158 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
17159 ret void
17160 }
17162 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
17163 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17164 ; GFX6: ; %bb.0: ; %entry
17165 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17166 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
17167 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
17168 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
17169 ; GFX6-NEXT: s_mov_b32 s12, s5
17170 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
17171 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
17172 ; GFX6-NEXT: s_mov_b32 s11, -1
17173 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
17174 ; GFX6-NEXT: s_mov_b32 s5, s12
17175 ; GFX6-NEXT: s_mov_b32 s6, s11
17176 ; GFX6-NEXT: s_mov_b32 s7, s10
17177 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
17178 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
17179 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17180 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17181 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
17182 ; GFX6-NEXT: s_endpgm
17183 ;
17184 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17185 ; GFX7: ; %bb.0: ; %entry
17186 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
17187 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
17188 ; GFX7-NEXT: s_load_dword s7, s[4:5], 0x2
17189 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x3
17190 ; GFX7-NEXT: s_mov_b64 s[10:11], 16
17191 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17192 ; GFX7-NEXT: s_mov_b32 s4, s8
17193 ; GFX7-NEXT: s_mov_b32 s5, s9
17194 ; GFX7-NEXT: s_mov_b32 s9, s10
17195 ; GFX7-NEXT: s_mov_b32 s8, s11
17196 ; GFX7-NEXT: s_add_u32 s4, s4, s9
17197 ; GFX7-NEXT: s_addc_u32 s8, s5, s8
17198 ; GFX7-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
17199 ; GFX7-NEXT: s_mov_b32 s5, s8
17200 ; GFX7-NEXT: v_mov_b32_e32 v2, s7
17201 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17202 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17203 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17204 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17205 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17206 ; GFX7-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
17207 ; GFX7-NEXT: s_endpgm
17208 ;
17209 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17210 ; GFX10-WGP: ; %bb.0: ; %entry
17211 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
17212 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
17213 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17214 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
17215 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
17216 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17217 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17218 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
17219 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17220 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
17221 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17222 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17223 ; GFX10-WGP-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
17224 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17225 ; GFX10-WGP-NEXT: buffer_gl0_inv
17226 ; GFX10-WGP-NEXT: s_endpgm
17227 ;
17228 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17229 ; GFX10-CU: ; %bb.0: ; %entry
17230 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
17231 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
17232 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17233 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
17234 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
17235 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17236 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17237 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
17238 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17239 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
17240 ; GFX10-CU-NEXT: global_atomic_cmpswap v0, v[1:2], s[4:5] offset:16
17241 ; GFX10-CU-NEXT: s_endpgm
17242 ;
17243 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17244 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17245 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17246 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17247 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17248 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17249 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
17250 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
17251 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
17252 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
17253 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
17254 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
17255 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
17256 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17257 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
17258 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
17259 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17260 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
17261 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
17262 ; SKIP-CACHE-INV-NEXT: s_endpgm
17263 ;
17264 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17265 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17266 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17267 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17268 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17269 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17270 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17271 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17272 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17273 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17274 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17275 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17276 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
17277 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17278 ;
17279 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17280 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17281 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17282 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17283 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17284 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17285 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17286 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17287 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17288 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17289 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17290 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17291 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17292 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[4:5] offset:16
17293 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17294 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17295 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17296 ;
17297 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17298 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17299 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17300 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17301 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17302 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17303 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17304 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17305 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17306 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17307 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17308 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17309 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
17310 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17311 ;
17312 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17313 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17314 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17315 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17316 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17317 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17318 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17319 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17320 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17321 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17322 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17323 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17324 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17325 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v0, v[2:3], s[0:1] offset:16
17326 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17327 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
17328 ; GFX940-TGSPLIT-NEXT: s_endpgm
17329 ;
17330 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17331 ; GFX11-WGP: ; %bb.0: ; %entry
17332 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17333 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
17334 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17335 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17336 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17337 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17338 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
17339 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
17340 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17341 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
17342 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17343 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17344 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17345 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17346 ; GFX11-WGP-NEXT: buffer_gl0_inv
17347 ; GFX11-WGP-NEXT: s_endpgm
17348 ;
17349 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17350 ; GFX11-CU: ; %bb.0: ; %entry
17351 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17352 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
17353 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17354 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17355 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17356 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17357 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
17358 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
17359 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17360 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
17361 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17362 ; GFX11-CU-NEXT: s_endpgm
17363 ;
17364 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17365 ; GFX12-WGP: ; %bb.0: ; %entry
17366 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17367 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
17368 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17369 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17370 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17371 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17372 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
17373 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
17374 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17375 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
17376 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
17377 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17378 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17379 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17380 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17381 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SE
17382 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
17383 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
17384 ; GFX12-WGP-NEXT: s_endpgm
17385 ;
17386 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
17387 ; GFX12-CU: ; %bb.0: ; %entry
17388 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17389 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
17390 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17391 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17392 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17393 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17394 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
17395 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
17396 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17397 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
17398 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
17399 ; GFX12-CU-NEXT: s_endpgm
17400 ptr addrspace(1) %out, i32 %in, i32 %old) {
17401 entry:
17402 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
17403 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
17404 ret void
17405 }
17407 define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg(
17408 ; GFX6-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17409 ; GFX6: ; %bb.0: ; %entry
17410 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17411 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
17412 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
17413 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
17414 ; GFX6-NEXT: s_mov_b32 s12, s5
17415 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
17416 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
17417 ; GFX6-NEXT: s_mov_b32 s11, -1
17418 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
17419 ; GFX6-NEXT: s_mov_b32 s5, s12
17420 ; GFX6-NEXT: s_mov_b32 s6, s11
17421 ; GFX6-NEXT: s_mov_b32 s7, s10
17422 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
17423 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
17424 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17425 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17426 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
17427 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
17428 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17429 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
17430 ; GFX6-NEXT: s_endpgm
17431 ;
17432 ; GFX7-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17433 ; GFX7: ; %bb.0: ; %entry
17434 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17435 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17436 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17437 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17438 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17439 ; GFX7-NEXT: s_mov_b32 s6, s4
17440 ; GFX7-NEXT: s_mov_b32 s7, s5
17441 ; GFX7-NEXT: s_mov_b32 s11, s12
17442 ; GFX7-NEXT: s_mov_b32 s10, s13
17443 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17444 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17445 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17446 ; GFX7-NEXT: s_mov_b32 s7, s10
17447 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17448 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17449 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17450 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17451 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17452 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17453 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17454 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17455 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17456 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17457 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17458 ; GFX7-NEXT: s_endpgm
17459 ;
17460 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17461 ; GFX10-WGP: ; %bb.0: ; %entry
17462 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
17463 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
17464 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17465 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
17466 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
17467 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17468 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17469 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
17470 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17471 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
17472 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
17473 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17474 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
17475 ; GFX10-WGP-NEXT: s_endpgm
17476 ;
17477 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17478 ; GFX10-CU: ; %bb.0: ; %entry
17479 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
17480 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
17481 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17482 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
17483 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
17484 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17485 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17486 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
17487 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17488 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
17489 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
17490 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17491 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
17492 ; GFX10-CU-NEXT: s_endpgm
17493 ;
17494 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17495 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17496 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17497 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17498 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17499 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17500 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
17501 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
17502 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
17503 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
17504 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
17505 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
17506 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
17507 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17508 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
17509 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
17510 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17511 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
17512 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
17513 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
17514 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17515 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
17516 ; SKIP-CACHE-INV-NEXT: s_endpgm
17517 ;
17518 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17519 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17520 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17521 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17522 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17523 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17524 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17525 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17526 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17527 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17528 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17529 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17530 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
17531 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17532 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
17533 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17534 ;
17535 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17536 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17537 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17538 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17539 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17540 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17541 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17542 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17543 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17544 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17545 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17546 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17547 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
17548 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17549 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
17550 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17551 ;
17552 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17553 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17554 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17555 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17556 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17557 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17558 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17559 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17560 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17561 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17562 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17563 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17564 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
17565 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17566 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
17567 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17568 ;
17569 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17570 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17571 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17572 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17573 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17574 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17575 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17576 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17577 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17578 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17579 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17580 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17581 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
17582 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17583 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
17584 ; GFX940-TGSPLIT-NEXT: s_endpgm
17585 ;
17586 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17587 ; GFX11-WGP: ; %bb.0: ; %entry
17588 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17589 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
17590 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17591 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17592 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17593 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17594 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
17595 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
17596 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17597 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
17598 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
17599 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17600 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
17601 ; GFX11-WGP-NEXT: s_endpgm
17602 ;
17603 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17604 ; GFX11-CU: ; %bb.0: ; %entry
17605 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17606 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
17607 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17608 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17609 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17610 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17611 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
17612 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
17613 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17614 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
17615 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
17616 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
17617 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
17618 ; GFX11-CU-NEXT: s_endpgm
17619 ;
17620 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17621 ; GFX12-WGP: ; %bb.0: ; %entry
17622 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17623 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
17624 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17625 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17626 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17627 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17628 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
17629 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
17630 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17631 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
17632 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
17633 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17634 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
17635 ; GFX12-WGP-NEXT: s_endpgm
17636 ;
17637 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
17638 ; GFX12-CU: ; %bb.0: ; %entry
17639 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17640 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
17641 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17642 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17643 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17644 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17645 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
17646 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
17647 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17648 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
17649 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
17650 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17651 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
17652 ; GFX12-CU-NEXT: s_endpgm
17653 ptr addrspace(1) %out, i32 %in, i32 %old) {
17654 entry:
17655 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
17656 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
17657 %val0 = extractvalue { i32, i1 } %val, 0
17658 store i32 %val0, ptr addrspace(1) %out, align 4
17659 ret void
17660 }
17662 define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
17663 ; GFX6-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17664 ; GFX6: ; %bb.0: ; %entry
17665 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17666 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
17667 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
17668 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
17669 ; GFX6-NEXT: s_mov_b32 s12, s5
17670 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
17671 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
17672 ; GFX6-NEXT: s_mov_b32 s11, -1
17673 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
17674 ; GFX6-NEXT: s_mov_b32 s5, s12
17675 ; GFX6-NEXT: s_mov_b32 s6, s11
17676 ; GFX6-NEXT: s_mov_b32 s7, s10
17677 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
17678 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
17679 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17680 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17681 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
17682 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
17683 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17684 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
17685 ; GFX6-NEXT: s_endpgm
17686 ;
17687 ; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17688 ; GFX7: ; %bb.0: ; %entry
17689 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17690 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17691 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17692 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17693 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17694 ; GFX7-NEXT: s_mov_b32 s6, s4
17695 ; GFX7-NEXT: s_mov_b32 s7, s5
17696 ; GFX7-NEXT: s_mov_b32 s11, s12
17697 ; GFX7-NEXT: s_mov_b32 s10, s13
17698 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17699 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17700 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17701 ; GFX7-NEXT: s_mov_b32 s7, s10
17702 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17703 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17704 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17705 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17706 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17707 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17708 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17709 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17710 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17711 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17712 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17713 ; GFX7-NEXT: s_endpgm
17714 ;
17715 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17716 ; GFX10-WGP: ; %bb.0: ; %entry
17717 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
17718 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
17719 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17720 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
17721 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
17722 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17723 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17724 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
17725 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17726 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
17727 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
17728 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17729 ; GFX10-WGP-NEXT: buffer_gl0_inv
17730 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
17731 ; GFX10-WGP-NEXT: s_endpgm
17732 ;
17733 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17734 ; GFX10-CU: ; %bb.0: ; %entry
17735 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
17736 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
17737 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17738 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
17739 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
17740 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
17741 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
17742 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
17743 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17744 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
17745 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
17746 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
17747 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
17748 ; GFX10-CU-NEXT: s_endpgm
17749 ;
17750 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17751 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
17752 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
17753 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
17754 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
17755 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
17756 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
17757 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
17758 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
17759 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
17760 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
17761 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
17762 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
17763 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
17764 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
17765 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
17766 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17767 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
17768 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
17769 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
17770 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
17771 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
17772 ; SKIP-CACHE-INV-NEXT: s_endpgm
17773 ;
17774 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17775 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
17776 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17777 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17778 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17779 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17780 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17781 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17782 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17783 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17784 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17785 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17786 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
17787 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17788 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
17789 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
17790 ;
17791 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17792 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
17793 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
17794 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17795 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17796 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
17797 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
17798 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17799 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
17800 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
17801 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17802 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17803 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
17804 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17805 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
17806 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
17807 ; GFX90A-TGSPLIT-NEXT: s_endpgm
17808 ;
17809 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17810 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
17811 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17812 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17813 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17814 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17815 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17816 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17817 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17818 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17819 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17820 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17821 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
17822 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
17823 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
17824 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
17825 ;
17826 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17827 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
17828 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
17829 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
17830 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
17831 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
17832 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
17833 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
17834 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
17835 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
17836 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17837 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
17838 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
17839 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
17840 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
17841 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
17842 ; GFX940-TGSPLIT-NEXT: s_endpgm
17843 ;
17844 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17845 ; GFX11-WGP: ; %bb.0: ; %entry
17846 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17847 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
17848 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17849 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17850 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17851 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
17852 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
17853 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
17854 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17855 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
17856 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
17857 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
17858 ; GFX11-WGP-NEXT: buffer_gl0_inv
17859 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
17860 ; GFX11-WGP-NEXT: s_endpgm
17861 ;
17862 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17863 ; GFX11-CU: ; %bb.0: ; %entry
17864 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17865 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
17866 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17867 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17868 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17869 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
17870 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
17871 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
17872 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17873 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
17874 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
17875 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
17876 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
17877 ; GFX11-CU-NEXT: s_endpgm
17878 ;
17879 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17880 ; GFX12-WGP: ; %bb.0: ; %entry
17881 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
17882 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
17883 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17884 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
17885 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
17886 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
17887 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
17888 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
17889 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17890 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
17891 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
17892 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
17893 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
17894 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
17895 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
17896 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
17897 ; GFX12-WGP-NEXT: s_endpgm
17898 ;
17899 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
17900 ; GFX12-CU: ; %bb.0: ; %entry
17901 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
17902 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
17903 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
17904 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
17905 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
17906 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
17907 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
17908 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
17909 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17910 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
17911 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
17912 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
17913 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
17914 ; GFX12-CU-NEXT: s_endpgm
17915 ptr addrspace(1) %out, i32 %in, i32 %old) {
17916 entry:
17917 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
17918 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
17919 %val0 = extractvalue { i32, i1 } %val, 0
17920 store i32 %val0, ptr addrspace(1) %out, align 4
17921 ret void
17922 }
17924 define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_ret_cmpxchg(
17925 ; GFX6-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
17926 ; GFX6: ; %bb.0: ; %entry
17927 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17928 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
17929 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
17930 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
17931 ; GFX6-NEXT: s_mov_b32 s12, s5
17932 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
17933 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
17934 ; GFX6-NEXT: s_mov_b32 s11, -1
17935 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
17936 ; GFX6-NEXT: s_mov_b32 s5, s12
17937 ; GFX6-NEXT: s_mov_b32 s6, s11
17938 ; GFX6-NEXT: s_mov_b32 s7, s10
17939 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
17940 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
17941 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
17942 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
17943 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
17944 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
17945 ; GFX6-NEXT: s_waitcnt vmcnt(0)
17946 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
17947 ; GFX6-NEXT: s_endpgm
17948 ;
17949 ; GFX7-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
17950 ; GFX7: ; %bb.0: ; %entry
17951 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
17952 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
17953 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
17954 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
17955 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17956 ; GFX7-NEXT: s_mov_b32 s6, s4
17957 ; GFX7-NEXT: s_mov_b32 s7, s5
17958 ; GFX7-NEXT: s_mov_b32 s11, s12
17959 ; GFX7-NEXT: s_mov_b32 s10, s13
17960 ; GFX7-NEXT: s_add_u32 s6, s6, s11
17961 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
17962 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
17963 ; GFX7-NEXT: s_mov_b32 s7, s10
17964 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
17965 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
17966 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
17967 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
17968 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
17969 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
17970 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
17971 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
17972 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
17973 ; GFX7-NEXT: s_waitcnt vmcnt(0)
17974 ; GFX7-NEXT: flat_store_dword v[0:1], v2
17975 ; GFX7-NEXT: s_endpgm
17976 ;
17977 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
17978 ; GFX10-WGP: ; %bb.0: ; %entry
17979 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
17980 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
17981 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
17982 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
17983 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
17984 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
17985 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
17986 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
17987 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
17988 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
17989 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17990 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
17991 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
17992 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
17993 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
17994 ; GFX10-WGP-NEXT: s_endpgm
17995 ;
17996 ; GFX10-CU-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
17997 ; GFX10-CU: ; %bb.0: ; %entry
17998 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
17999 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
18000 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18001 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
18002 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
18003 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18004 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18005 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
18006 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18007 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
18008 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18009 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18010 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
18011 ; GFX10-CU-NEXT: s_endpgm
18012 ;
18013 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18014 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18015 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18016 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18017 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18018 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18019 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
18020 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
18021 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
18022 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
18023 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
18024 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
18025 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
18026 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18027 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
18028 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
18029 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18030 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
18031 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
18032 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18033 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18034 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
18035 ; SKIP-CACHE-INV-NEXT: s_endpgm
18036 ;
18037 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18038 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18039 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18040 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18041 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18042 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18043 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18044 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18045 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18046 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18047 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18048 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18049 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18050 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18051 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18052 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18053 ;
18054 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18055 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18056 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18057 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18058 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18059 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18060 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18061 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18062 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18063 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18064 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18065 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18066 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18067 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18068 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18069 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18070 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18071 ;
18072 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18073 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18074 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18075 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18076 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18077 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18078 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18079 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18080 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18081 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18082 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18083 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18084 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18085 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18086 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18087 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18088 ;
18089 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18090 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18091 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18092 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18093 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18094 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18095 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18096 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18097 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18098 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18099 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18100 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18101 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18102 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18103 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18104 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18105 ; GFX940-TGSPLIT-NEXT: s_endpgm
18106 ;
18107 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18108 ; GFX11-WGP: ; %bb.0: ; %entry
18109 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18110 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
18111 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18112 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18113 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18114 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18115 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
18116 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
18117 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18118 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
18119 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18120 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18121 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18122 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18123 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18124 ; GFX11-WGP-NEXT: s_endpgm
18125 ;
18126 ; GFX11-CU-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18127 ; GFX11-CU: ; %bb.0: ; %entry
18128 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18129 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
18130 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18131 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18132 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18133 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18134 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
18135 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
18136 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18137 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
18138 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18139 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18140 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18141 ; GFX11-CU-NEXT: s_endpgm
18142 ;
18143 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18144 ; GFX12-WGP: ; %bb.0: ; %entry
18145 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18146 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
18147 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18148 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18149 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18150 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18151 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
18152 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
18153 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18154 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
18155 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
18156 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18157 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18158 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18159 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18160 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18161 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18162 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18163 ; GFX12-WGP-NEXT: s_endpgm
18164 ;
18165 ; GFX12-CU-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
18166 ; GFX12-CU: ; %bb.0: ; %entry
18167 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18168 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
18169 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18170 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18171 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18172 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18173 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
18174 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
18175 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18176 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
18177 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
18178 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18179 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18180 ; GFX12-CU-NEXT: s_endpgm
18181 ptr addrspace(1) %out, i32 %in, i32 %old) {
18182 entry:
18183 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
18184 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
18185 %val0 = extractvalue { i32, i1 } %val, 0
18186 store i32 %val0, ptr addrspace(1) %out, align 4
18187 ret void
18188 }
18190 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
18191 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18192 ; GFX6: ; %bb.0: ; %entry
18193 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18194 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
18195 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
18196 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
18197 ; GFX6-NEXT: s_mov_b32 s12, s5
18198 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
18199 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
18200 ; GFX6-NEXT: s_mov_b32 s11, -1
18201 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
18202 ; GFX6-NEXT: s_mov_b32 s5, s12
18203 ; GFX6-NEXT: s_mov_b32 s6, s11
18204 ; GFX6-NEXT: s_mov_b32 s7, s10
18205 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
18206 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
18207 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18208 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
18209 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
18210 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18211 ; GFX6-NEXT: s_waitcnt vmcnt(0)
18212 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
18213 ; GFX6-NEXT: s_endpgm
18214 ;
18215 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18216 ; GFX7: ; %bb.0: ; %entry
18217 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18218 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18219 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18220 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18221 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18222 ; GFX7-NEXT: s_mov_b32 s6, s4
18223 ; GFX7-NEXT: s_mov_b32 s7, s5
18224 ; GFX7-NEXT: s_mov_b32 s11, s12
18225 ; GFX7-NEXT: s_mov_b32 s10, s13
18226 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18227 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18228 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18229 ; GFX7-NEXT: s_mov_b32 s7, s10
18230 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18231 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18232 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18233 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18234 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18235 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18236 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18237 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18238 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18239 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18240 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18241 ; GFX7-NEXT: s_endpgm
18242 ;
18243 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18244 ; GFX10-WGP: ; %bb.0: ; %entry
18245 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
18246 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
18247 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18248 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
18249 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
18250 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18251 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18252 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
18253 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18254 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
18255 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18256 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18257 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18258 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18259 ; GFX10-WGP-NEXT: buffer_gl0_inv
18260 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
18261 ; GFX10-WGP-NEXT: s_endpgm
18262 ;
18263 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18264 ; GFX10-CU: ; %bb.0: ; %entry
18265 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
18266 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
18267 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18268 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
18269 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
18270 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18271 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18272 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
18273 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18274 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
18275 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18276 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18277 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
18278 ; GFX10-CU-NEXT: s_endpgm
18279 ;
18280 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18281 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18282 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18283 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18284 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18285 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18286 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
18287 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
18288 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
18289 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
18290 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
18291 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
18292 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
18293 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18294 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
18295 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
18296 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18297 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
18298 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
18299 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18300 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18301 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
18302 ; SKIP-CACHE-INV-NEXT: s_endpgm
18303 ;
18304 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18305 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18306 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18307 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18308 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18309 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18310 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18311 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18312 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18313 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18314 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18315 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18316 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18317 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18318 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18319 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18320 ;
18321 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18322 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18323 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18324 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18325 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18326 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18327 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18328 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18329 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18330 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18331 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18332 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18333 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18334 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18335 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18336 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18337 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18338 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18339 ;
18340 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18341 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18342 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18343 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18344 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18345 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18346 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18347 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18348 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18349 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18350 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18351 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18352 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18353 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18354 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18355 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18356 ;
18357 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18358 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18359 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18360 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18361 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18362 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18363 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18364 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18365 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18366 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18367 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18368 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18369 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18370 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18371 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18372 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18373 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18374 ; GFX940-TGSPLIT-NEXT: s_endpgm
18375 ;
18376 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18377 ; GFX11-WGP: ; %bb.0: ; %entry
18378 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18379 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
18380 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18381 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18382 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18383 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18384 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
18385 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
18386 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18387 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
18388 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18389 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18390 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18391 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18392 ; GFX11-WGP-NEXT: buffer_gl0_inv
18393 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18394 ; GFX11-WGP-NEXT: s_endpgm
18395 ;
18396 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18397 ; GFX11-CU: ; %bb.0: ; %entry
18398 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18399 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
18400 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18401 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18402 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18403 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18404 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
18405 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
18406 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18407 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
18408 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18409 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18410 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18411 ; GFX11-CU-NEXT: s_endpgm
18412 ;
18413 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18414 ; GFX12-WGP: ; %bb.0: ; %entry
18415 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18416 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
18417 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18418 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18419 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18420 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18421 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
18422 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
18423 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18424 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
18425 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
18426 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18427 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18428 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18429 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18430 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18431 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18432 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18433 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18434 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18435 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18436 ; GFX12-WGP-NEXT: s_endpgm
18437 ;
18438 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
18439 ; GFX12-CU: ; %bb.0: ; %entry
18440 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18441 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
18442 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18443 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18444 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18445 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18446 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
18447 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
18448 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18449 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
18450 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
18451 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18452 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18453 ; GFX12-CU-NEXT: s_endpgm
18454 ptr addrspace(1) %out, i32 %in, i32 %old) {
18455 entry:
18456 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
18457 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
18458 %val0 = extractvalue { i32, i1 } %val, 0
18459 store i32 %val0, ptr addrspace(1) %out, align 4
18460 ret void
18461 }
18463 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
18464 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18465 ; GFX6: ; %bb.0: ; %entry
18466 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18467 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
18468 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
18469 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
18470 ; GFX6-NEXT: s_mov_b32 s12, s5
18471 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
18472 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
18473 ; GFX6-NEXT: s_mov_b32 s11, -1
18474 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
18475 ; GFX6-NEXT: s_mov_b32 s5, s12
18476 ; GFX6-NEXT: s_mov_b32 s6, s11
18477 ; GFX6-NEXT: s_mov_b32 s7, s10
18478 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
18479 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
18480 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18481 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
18482 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
18483 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18484 ; GFX6-NEXT: s_waitcnt vmcnt(0)
18485 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
18486 ; GFX6-NEXT: s_endpgm
18487 ;
18488 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18489 ; GFX7: ; %bb.0: ; %entry
18490 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18491 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18492 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18493 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18494 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18495 ; GFX7-NEXT: s_mov_b32 s6, s4
18496 ; GFX7-NEXT: s_mov_b32 s7, s5
18497 ; GFX7-NEXT: s_mov_b32 s11, s12
18498 ; GFX7-NEXT: s_mov_b32 s10, s13
18499 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18500 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18501 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18502 ; GFX7-NEXT: s_mov_b32 s7, s10
18503 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18504 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18505 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18506 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18507 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18508 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18509 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18510 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18511 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18512 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18513 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18514 ; GFX7-NEXT: s_endpgm
18515 ;
18516 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18517 ; GFX10-WGP: ; %bb.0: ; %entry
18518 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
18519 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
18520 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18521 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
18522 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
18523 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18524 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18525 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
18526 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18527 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
18528 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18529 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18530 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18531 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18532 ; GFX10-WGP-NEXT: buffer_gl0_inv
18533 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
18534 ; GFX10-WGP-NEXT: s_endpgm
18535 ;
18536 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18537 ; GFX10-CU: ; %bb.0: ; %entry
18538 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
18539 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
18540 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18541 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
18542 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
18543 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18544 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18545 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
18546 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18547 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
18548 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18549 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18550 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
18551 ; GFX10-CU-NEXT: s_endpgm
18552 ;
18553 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18554 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18555 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18556 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18557 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18558 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18559 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
18560 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
18561 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
18562 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
18563 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
18564 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
18565 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
18566 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18567 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
18568 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
18569 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18570 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
18571 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
18572 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18573 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18574 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
18575 ; SKIP-CACHE-INV-NEXT: s_endpgm
18576 ;
18577 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18578 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18579 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18580 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18581 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18582 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18583 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18584 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18585 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18586 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18587 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18588 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18589 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18590 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18591 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18592 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18593 ;
18594 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18595 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18596 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18597 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18598 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18599 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18600 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18601 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18602 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18603 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18604 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18605 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18606 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18607 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18608 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18609 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18610 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18611 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18612 ;
18613 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18614 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18615 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18616 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18617 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18618 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18619 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18620 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18621 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18622 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18623 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18624 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18625 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18626 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18627 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18628 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18629 ;
18630 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18631 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18632 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18633 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18634 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18635 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18636 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18637 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18638 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18639 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18640 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18641 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18642 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18643 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18644 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18645 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18646 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18647 ; GFX940-TGSPLIT-NEXT: s_endpgm
18648 ;
18649 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18650 ; GFX11-WGP: ; %bb.0: ; %entry
18651 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18652 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
18653 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18654 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18655 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18656 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18657 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
18658 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
18659 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18660 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
18661 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18662 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
18663 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18664 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18665 ; GFX11-WGP-NEXT: buffer_gl0_inv
18666 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18667 ; GFX11-WGP-NEXT: s_endpgm
18668 ;
18669 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18670 ; GFX11-CU: ; %bb.0: ; %entry
18671 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18672 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
18673 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18674 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18675 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18676 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18677 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
18678 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
18679 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18680 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
18681 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18682 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18683 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18684 ; GFX11-CU-NEXT: s_endpgm
18685 ;
18686 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18687 ; GFX12-WGP: ; %bb.0: ; %entry
18688 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18689 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
18690 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18691 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18692 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18693 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18694 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
18695 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
18696 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18697 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
18698 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
18699 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18700 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18701 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18702 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
18703 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18704 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18705 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18706 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18707 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18708 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18709 ; GFX12-WGP-NEXT: s_endpgm
18710 ;
18711 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
18712 ; GFX12-CU: ; %bb.0: ; %entry
18713 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18714 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
18715 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18716 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18717 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18718 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18719 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
18720 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
18721 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18722 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
18723 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
18724 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18725 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18726 ; GFX12-CU-NEXT: s_endpgm
18727 ptr addrspace(1) %out, i32 %in, i32 %old) {
18728 entry:
18729 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
18730 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
18731 %val0 = extractvalue { i32, i1 } %val, 0
18732 store i32 %val0, ptr addrspace(1) %out, align 4
18733 ret void
18734 }
18736 define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
18737 ; GFX6-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18738 ; GFX6: ; %bb.0: ; %entry
18739 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18740 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
18741 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
18742 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
18743 ; GFX6-NEXT: s_mov_b32 s12, s5
18744 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
18745 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
18746 ; GFX6-NEXT: s_mov_b32 s11, -1
18747 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
18748 ; GFX6-NEXT: s_mov_b32 s5, s12
18749 ; GFX6-NEXT: s_mov_b32 s6, s11
18750 ; GFX6-NEXT: s_mov_b32 s7, s10
18751 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
18752 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
18753 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18754 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
18755 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
18756 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18757 ; GFX6-NEXT: s_waitcnt vmcnt(0)
18758 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
18759 ; GFX6-NEXT: s_endpgm
18760 ;
18761 ; GFX7-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18762 ; GFX7: ; %bb.0: ; %entry
18763 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
18764 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
18765 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
18766 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
18767 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
18768 ; GFX7-NEXT: s_mov_b32 s6, s4
18769 ; GFX7-NEXT: s_mov_b32 s7, s5
18770 ; GFX7-NEXT: s_mov_b32 s11, s12
18771 ; GFX7-NEXT: s_mov_b32 s10, s13
18772 ; GFX7-NEXT: s_add_u32 s6, s6, s11
18773 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
18774 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
18775 ; GFX7-NEXT: s_mov_b32 s7, s10
18776 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
18777 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
18778 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18779 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
18780 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18781 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
18782 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
18783 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
18784 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
18785 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18786 ; GFX7-NEXT: flat_store_dword v[0:1], v2
18787 ; GFX7-NEXT: s_endpgm
18788 ;
18789 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18790 ; GFX10-WGP: ; %bb.0: ; %entry
18791 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
18792 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
18793 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18794 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
18795 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
18796 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
18797 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
18798 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
18799 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18800 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
18801 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18802 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
18803 ; GFX10-WGP-NEXT: buffer_gl0_inv
18804 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
18805 ; GFX10-WGP-NEXT: s_endpgm
18806 ;
18807 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18808 ; GFX10-CU: ; %bb.0: ; %entry
18809 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
18810 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
18811 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18812 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
18813 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
18814 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
18815 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
18816 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
18817 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18818 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
18819 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
18820 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
18821 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
18822 ; GFX10-CU-NEXT: s_endpgm
18823 ;
18824 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18825 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
18826 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
18827 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
18828 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
18829 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
18830 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
18831 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
18832 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
18833 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
18834 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
18835 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
18836 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
18837 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
18838 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
18839 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
18840 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
18841 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
18842 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
18843 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
18844 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
18845 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
18846 ; SKIP-CACHE-INV-NEXT: s_endpgm
18847 ;
18848 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18849 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
18850 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18851 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18852 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18853 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18854 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18855 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18856 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18857 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18858 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18859 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18860 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18861 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18862 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18863 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
18864 ;
18865 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18866 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
18867 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
18868 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18869 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
18870 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
18871 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
18872 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18873 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
18874 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
18875 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18876 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18877 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
18878 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18879 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
18880 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
18881 ; GFX90A-TGSPLIT-NEXT: s_endpgm
18882 ;
18883 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18884 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
18885 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18886 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18887 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18888 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18889 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18890 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18891 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18892 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18893 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18894 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18895 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18896 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
18897 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18898 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
18899 ;
18900 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18901 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
18902 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
18903 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
18904 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
18905 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
18906 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
18907 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
18908 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
18909 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
18910 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
18911 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
18912 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
18913 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
18914 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
18915 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
18916 ; GFX940-TGSPLIT-NEXT: s_endpgm
18917 ;
18918 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18919 ; GFX11-WGP: ; %bb.0: ; %entry
18920 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18921 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
18922 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18923 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18924 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18925 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
18926 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
18927 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
18928 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18929 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
18930 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18931 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
18932 ; GFX11-WGP-NEXT: buffer_gl0_inv
18933 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18934 ; GFX11-WGP-NEXT: s_endpgm
18935 ;
18936 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18937 ; GFX11-CU: ; %bb.0: ; %entry
18938 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18939 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
18940 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18941 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18942 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18943 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
18944 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
18945 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
18946 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18947 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
18948 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
18949 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
18950 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18951 ; GFX11-CU-NEXT: s_endpgm
18952 ;
18953 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18954 ; GFX12-WGP: ; %bb.0: ; %entry
18955 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
18956 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
18957 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18958 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
18959 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
18960 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
18961 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
18962 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
18963 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18964 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
18965 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
18966 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
18967 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
18968 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
18969 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
18970 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
18971 ; GFX12-WGP-NEXT: s_endpgm
18972 ;
18973 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
18974 ; GFX12-CU: ; %bb.0: ; %entry
18975 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
18976 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
18977 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
18978 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
18979 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
18980 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
18981 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
18982 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
18983 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
18984 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
18985 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
18986 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
18987 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
18988 ; GFX12-CU-NEXT: s_endpgm
18989 ptr addrspace(1) %out, i32 %in, i32 %old) {
18990 entry:
18991 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
18992 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire
18993 %val0 = extractvalue { i32, i1 } %val, 0
18994 store i32 %val0, ptr addrspace(1) %out, align 4
18995 ret void
18996 }
18998 define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
18999 ; GFX6-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19000 ; GFX6: ; %bb.0: ; %entry
19001 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19002 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
19003 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
19004 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
19005 ; GFX6-NEXT: s_mov_b32 s12, s5
19006 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
19007 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
19008 ; GFX6-NEXT: s_mov_b32 s11, -1
19009 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
19010 ; GFX6-NEXT: s_mov_b32 s5, s12
19011 ; GFX6-NEXT: s_mov_b32 s6, s11
19012 ; GFX6-NEXT: s_mov_b32 s7, s10
19013 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
19014 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
19015 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19016 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
19017 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
19018 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19019 ; GFX6-NEXT: s_waitcnt vmcnt(0)
19020 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
19021 ; GFX6-NEXT: s_endpgm
19022 ;
19023 ; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19024 ; GFX7: ; %bb.0: ; %entry
19025 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19026 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19027 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19028 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19029 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19030 ; GFX7-NEXT: s_mov_b32 s6, s4
19031 ; GFX7-NEXT: s_mov_b32 s7, s5
19032 ; GFX7-NEXT: s_mov_b32 s11, s12
19033 ; GFX7-NEXT: s_mov_b32 s10, s13
19034 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19035 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19036 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19037 ; GFX7-NEXT: s_mov_b32 s7, s10
19038 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19039 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19040 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19041 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19042 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19043 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19044 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19045 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19046 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19047 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19048 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19049 ; GFX7-NEXT: s_endpgm
19050 ;
19051 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19052 ; GFX10-WGP: ; %bb.0: ; %entry
19053 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
19054 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
19055 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19056 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
19057 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
19058 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19059 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19060 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
19061 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19062 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
19063 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19064 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19065 ; GFX10-WGP-NEXT: buffer_gl0_inv
19066 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
19067 ; GFX10-WGP-NEXT: s_endpgm
19068 ;
19069 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19070 ; GFX10-CU: ; %bb.0: ; %entry
19071 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
19072 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
19073 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19074 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
19075 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
19076 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19077 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19078 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
19079 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19080 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
19081 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19082 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19083 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
19084 ; GFX10-CU-NEXT: s_endpgm
19085 ;
19086 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19087 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19088 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19089 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19090 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19091 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19092 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
19093 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
19094 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
19095 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
19096 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
19097 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
19098 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
19099 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19100 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
19101 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
19102 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19103 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
19104 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
19105 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19106 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19107 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
19108 ; SKIP-CACHE-INV-NEXT: s_endpgm
19109 ;
19110 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19111 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19112 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19113 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19114 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19115 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19116 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19117 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19118 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19119 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19120 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19121 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19122 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19123 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19124 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19125 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19126 ;
19127 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19128 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19129 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19130 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19131 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19132 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19133 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19134 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19135 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19136 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19137 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19138 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19139 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19140 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19141 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19142 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19143 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19144 ;
19145 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19146 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19147 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19148 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19149 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19150 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19151 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19152 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19153 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19154 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19155 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19156 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19157 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19158 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19159 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19160 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19161 ;
19162 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19163 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19164 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19165 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19166 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19167 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19168 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19169 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19170 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19171 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19172 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19173 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19174 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19175 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19176 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19177 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19178 ; GFX940-TGSPLIT-NEXT: s_endpgm
19179 ;
19180 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19181 ; GFX11-WGP: ; %bb.0: ; %entry
19182 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19183 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
19184 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19185 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19186 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19187 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19188 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
19189 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
19190 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19191 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
19192 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19193 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19194 ; GFX11-WGP-NEXT: buffer_gl0_inv
19195 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19196 ; GFX11-WGP-NEXT: s_endpgm
19197 ;
19198 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19199 ; GFX11-CU: ; %bb.0: ; %entry
19200 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19201 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
19202 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19203 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19204 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19205 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19206 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
19207 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
19208 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19209 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
19210 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19211 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19212 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19213 ; GFX11-CU-NEXT: s_endpgm
19214 ;
19215 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19216 ; GFX12-WGP: ; %bb.0: ; %entry
19217 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19218 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
19219 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19220 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19221 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19222 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19223 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
19224 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
19225 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19226 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
19227 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19228 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19229 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19230 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19231 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19232 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19233 ; GFX12-WGP-NEXT: s_endpgm
19234 ;
19235 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
19236 ; GFX12-CU: ; %bb.0: ; %entry
19237 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19238 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
19239 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19240 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19241 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19242 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19243 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
19244 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
19245 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19246 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
19247 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
19248 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19249 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19250 ; GFX12-CU-NEXT: s_endpgm
19251 ptr addrspace(1) %out, i32 %in, i32 %old) {
19252 entry:
19253 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
19254 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
19255 %val0 = extractvalue { i32, i1 } %val, 0
19256 store i32 %val0, ptr addrspace(1) %out, align 4
19257 ret void
19258 }
19260 define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
19261 ; GFX6-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19262 ; GFX6: ; %bb.0: ; %entry
19263 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19264 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
19265 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
19266 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
19267 ; GFX6-NEXT: s_mov_b32 s12, s5
19268 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
19269 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
19270 ; GFX6-NEXT: s_mov_b32 s11, -1
19271 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
19272 ; GFX6-NEXT: s_mov_b32 s5, s12
19273 ; GFX6-NEXT: s_mov_b32 s6, s11
19274 ; GFX6-NEXT: s_mov_b32 s7, s10
19275 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
19276 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
19277 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19278 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
19279 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
19280 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19281 ; GFX6-NEXT: s_waitcnt vmcnt(0)
19282 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
19283 ; GFX6-NEXT: s_endpgm
19284 ;
19285 ; GFX7-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19286 ; GFX7: ; %bb.0: ; %entry
19287 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19288 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19289 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19290 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19291 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19292 ; GFX7-NEXT: s_mov_b32 s6, s4
19293 ; GFX7-NEXT: s_mov_b32 s7, s5
19294 ; GFX7-NEXT: s_mov_b32 s11, s12
19295 ; GFX7-NEXT: s_mov_b32 s10, s13
19296 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19297 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19298 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19299 ; GFX7-NEXT: s_mov_b32 s7, s10
19300 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19301 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19302 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19303 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19304 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19305 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19306 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19307 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19308 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19309 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19310 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19311 ; GFX7-NEXT: s_endpgm
19312 ;
19313 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19314 ; GFX10-WGP: ; %bb.0: ; %entry
19315 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
19316 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
19317 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19318 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
19319 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
19320 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19321 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19322 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
19323 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19324 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
19325 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19326 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19327 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19328 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19329 ; GFX10-WGP-NEXT: buffer_gl0_inv
19330 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
19331 ; GFX10-WGP-NEXT: s_endpgm
19332 ;
19333 ; GFX10-CU-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19334 ; GFX10-CU: ; %bb.0: ; %entry
19335 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
19336 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
19337 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19338 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
19339 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
19340 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19341 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19342 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
19343 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19344 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
19345 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19346 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19347 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
19348 ; GFX10-CU-NEXT: s_endpgm
19349 ;
19350 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19351 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19352 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19353 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19354 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19355 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19356 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
19357 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
19358 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
19359 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
19360 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
19361 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
19362 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
19363 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19364 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
19365 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
19366 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19367 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
19368 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
19369 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19370 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19371 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
19372 ; SKIP-CACHE-INV-NEXT: s_endpgm
19373 ;
19374 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19375 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19376 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19377 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19378 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19379 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19380 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19381 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19382 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19383 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19384 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19385 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19386 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19387 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19388 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19389 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19390 ;
19391 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19392 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19393 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19394 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19395 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19396 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19397 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19398 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19399 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19400 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19401 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19402 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19403 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19404 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19405 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19406 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19407 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19408 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19409 ;
19410 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19411 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19412 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19413 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19414 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19415 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19416 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19417 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19418 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19419 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19420 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19421 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19422 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19423 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19424 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19425 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19426 ;
19427 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19428 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19429 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19430 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19431 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19432 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19433 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19434 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19435 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19436 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19437 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19438 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19439 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19440 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19441 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19442 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19443 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19444 ; GFX940-TGSPLIT-NEXT: s_endpgm
19445 ;
19446 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19447 ; GFX11-WGP: ; %bb.0: ; %entry
19448 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19449 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
19450 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19451 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19452 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19453 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19454 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
19455 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
19456 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19457 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
19458 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19459 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19460 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19461 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19462 ; GFX11-WGP-NEXT: buffer_gl0_inv
19463 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19464 ; GFX11-WGP-NEXT: s_endpgm
19465 ;
19466 ; GFX11-CU-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19467 ; GFX11-CU: ; %bb.0: ; %entry
19468 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19469 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
19470 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19471 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19472 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19473 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19474 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
19475 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
19476 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19477 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
19478 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19479 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19480 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19481 ; GFX11-CU-NEXT: s_endpgm
19482 ;
19483 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19484 ; GFX12-WGP: ; %bb.0: ; %entry
19485 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19486 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
19487 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19488 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19489 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19490 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19491 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
19492 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
19493 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19494 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
19495 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
19496 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19497 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19498 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19499 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19500 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19501 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19502 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19503 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19504 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19505 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19506 ; GFX12-WGP-NEXT: s_endpgm
19507 ;
19508 ; GFX12-CU-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
19509 ; GFX12-CU: ; %bb.0: ; %entry
19510 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19511 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
19512 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19513 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19514 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19515 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19516 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
19517 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
19518 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19519 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
19520 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
19521 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19522 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19523 ; GFX12-CU-NEXT: s_endpgm
19524 ptr addrspace(1) %out, i32 %in, i32 %old) {
19525 entry:
19526 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
19527 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
19528 %val0 = extractvalue { i32, i1 } %val, 0
19529 store i32 %val0, ptr addrspace(1) %out, align 4
19530 ret void
19531 }
19533 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
19534 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19535 ; GFX6: ; %bb.0: ; %entry
19536 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19537 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
19538 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
19539 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
19540 ; GFX6-NEXT: s_mov_b32 s12, s5
19541 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
19542 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
19543 ; GFX6-NEXT: s_mov_b32 s11, -1
19544 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
19545 ; GFX6-NEXT: s_mov_b32 s5, s12
19546 ; GFX6-NEXT: s_mov_b32 s6, s11
19547 ; GFX6-NEXT: s_mov_b32 s7, s10
19548 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
19549 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
19550 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19551 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
19552 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
19553 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19554 ; GFX6-NEXT: s_waitcnt vmcnt(0)
19555 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
19556 ; GFX6-NEXT: s_endpgm
19557 ;
19558 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19559 ; GFX7: ; %bb.0: ; %entry
19560 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19561 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19562 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19563 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19564 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19565 ; GFX7-NEXT: s_mov_b32 s6, s4
19566 ; GFX7-NEXT: s_mov_b32 s7, s5
19567 ; GFX7-NEXT: s_mov_b32 s11, s12
19568 ; GFX7-NEXT: s_mov_b32 s10, s13
19569 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19570 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19571 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19572 ; GFX7-NEXT: s_mov_b32 s7, s10
19573 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19574 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19575 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19576 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19577 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19578 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19579 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19580 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19581 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19582 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19583 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19584 ; GFX7-NEXT: s_endpgm
19585 ;
19586 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19587 ; GFX10-WGP: ; %bb.0: ; %entry
19588 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
19589 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
19590 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19591 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
19592 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
19593 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19594 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19595 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
19596 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19597 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
19598 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19599 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19600 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19601 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19602 ; GFX10-WGP-NEXT: buffer_gl0_inv
19603 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
19604 ; GFX10-WGP-NEXT: s_endpgm
19605 ;
19606 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19607 ; GFX10-CU: ; %bb.0: ; %entry
19608 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
19609 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
19610 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19611 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
19612 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
19613 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19614 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19615 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
19616 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19617 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
19618 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19619 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19620 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
19621 ; GFX10-CU-NEXT: s_endpgm
19622 ;
19623 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19624 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19625 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19626 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19627 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19628 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19629 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
19630 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
19631 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
19632 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
19633 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
19634 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
19635 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
19636 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19637 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
19638 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
19639 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19640 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
19641 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
19642 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19643 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19644 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
19645 ; SKIP-CACHE-INV-NEXT: s_endpgm
19646 ;
19647 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19648 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19649 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19650 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19651 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19652 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19653 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19654 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19655 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19656 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19657 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19658 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19659 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19660 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19661 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19662 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19663 ;
19664 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19665 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19666 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19667 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19668 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19669 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19670 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19671 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19672 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19673 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19674 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19675 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19676 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19677 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19678 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19679 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19680 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19681 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19682 ;
19683 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19684 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19685 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19686 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19687 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19688 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19689 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19690 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19691 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19692 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19693 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19694 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19695 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19696 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19697 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19698 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19699 ;
19700 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19701 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19702 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19703 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19704 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19705 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19706 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19707 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19708 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19709 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19710 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19711 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19712 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19713 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19714 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19715 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19716 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19717 ; GFX940-TGSPLIT-NEXT: s_endpgm
19718 ;
19719 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19720 ; GFX11-WGP: ; %bb.0: ; %entry
19721 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19722 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
19723 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19724 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19725 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19726 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
19727 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
19728 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
19729 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19730 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
19731 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19732 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19733 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19734 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
19735 ; GFX11-WGP-NEXT: buffer_gl0_inv
19736 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19737 ; GFX11-WGP-NEXT: s_endpgm
19738 ;
19739 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19740 ; GFX11-CU: ; %bb.0: ; %entry
19741 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19742 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
19743 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19744 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19745 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19746 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
19747 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
19748 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
19749 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19750 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
19751 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
19752 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
19753 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19754 ; GFX11-CU-NEXT: s_endpgm
19755 ;
19756 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19757 ; GFX12-WGP: ; %bb.0: ; %entry
19758 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19759 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
19760 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19761 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19762 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19763 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
19764 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
19765 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
19766 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19767 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
19768 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
19769 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19770 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19771 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19772 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
19773 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
19774 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
19775 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
19776 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
19777 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
19778 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
19779 ; GFX12-WGP-NEXT: s_endpgm
19780 ;
19781 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
19782 ; GFX12-CU: ; %bb.0: ; %entry
19783 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
19784 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
19785 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19786 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
19787 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
19788 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
19789 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
19790 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
19791 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19792 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
19793 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
19794 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
19795 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
19796 ; GFX12-CU-NEXT: s_endpgm
19797 ptr addrspace(1) %out, i32 %in, i32 %old) {
19798 entry:
19799 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
19800 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
19801 %val0 = extractvalue { i32, i1 } %val, 0
19802 store i32 %val0, ptr addrspace(1) %out, align 4
19803 ret void
19804 }
19806 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
19807 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19808 ; GFX6: ; %bb.0: ; %entry
19809 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19810 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
19811 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
19812 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
19813 ; GFX6-NEXT: s_mov_b32 s12, s5
19814 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
19815 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
19816 ; GFX6-NEXT: s_mov_b32 s11, -1
19817 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
19818 ; GFX6-NEXT: s_mov_b32 s5, s12
19819 ; GFX6-NEXT: s_mov_b32 s6, s11
19820 ; GFX6-NEXT: s_mov_b32 s7, s10
19821 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
19822 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
19823 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19824 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
19825 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
19826 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19827 ; GFX6-NEXT: s_waitcnt vmcnt(0)
19828 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
19829 ; GFX6-NEXT: s_endpgm
19830 ;
19831 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19832 ; GFX7: ; %bb.0: ; %entry
19833 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
19834 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
19835 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
19836 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
19837 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19838 ; GFX7-NEXT: s_mov_b32 s6, s4
19839 ; GFX7-NEXT: s_mov_b32 s7, s5
19840 ; GFX7-NEXT: s_mov_b32 s11, s12
19841 ; GFX7-NEXT: s_mov_b32 s10, s13
19842 ; GFX7-NEXT: s_add_u32 s6, s6, s11
19843 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
19844 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
19845 ; GFX7-NEXT: s_mov_b32 s7, s10
19846 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
19847 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
19848 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19849 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
19850 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
19851 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19852 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
19853 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
19854 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
19855 ; GFX7-NEXT: s_waitcnt vmcnt(0)
19856 ; GFX7-NEXT: flat_store_dword v[0:1], v2
19857 ; GFX7-NEXT: s_endpgm
19858 ;
19859 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19860 ; GFX10-WGP: ; %bb.0: ; %entry
19861 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
19862 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
19863 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19864 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
19865 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
19866 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
19867 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
19868 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
19869 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19870 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
19871 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19872 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
19873 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19874 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
19875 ; GFX10-WGP-NEXT: buffer_gl0_inv
19876 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
19877 ; GFX10-WGP-NEXT: s_endpgm
19878 ;
19879 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19880 ; GFX10-CU: ; %bb.0: ; %entry
19881 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
19882 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
19883 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19884 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
19885 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
19886 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
19887 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
19888 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
19889 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
19890 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
19891 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
19892 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
19893 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
19894 ; GFX10-CU-NEXT: s_endpgm
19895 ;
19896 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19897 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
19898 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
19899 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
19900 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
19901 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
19902 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
19903 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
19904 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
19905 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
19906 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
19907 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
19908 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
19909 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
19910 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
19911 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
19912 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
19913 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
19914 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
19915 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
19916 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
19917 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
19918 ; SKIP-CACHE-INV-NEXT: s_endpgm
19919 ;
19920 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19921 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
19922 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19923 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19924 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19925 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19926 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19927 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19928 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19929 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19930 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19931 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19932 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19933 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19934 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19935 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
19936 ;
19937 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19938 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
19939 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
19940 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19941 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
19942 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
19943 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
19944 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19945 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
19946 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
19947 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19948 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19949 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19950 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
19951 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19952 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
19953 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
19954 ; GFX90A-TGSPLIT-NEXT: s_endpgm
19955 ;
19956 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19957 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
19958 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19959 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19960 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19961 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19962 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19963 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19964 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19965 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19966 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19967 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19968 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19969 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
19970 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19971 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
19972 ;
19973 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19974 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
19975 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
19976 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
19977 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
19978 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
19979 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
19980 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
19981 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
19982 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
19983 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
19984 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
19985 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19986 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
19987 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
19988 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
19989 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
19990 ; GFX940-TGSPLIT-NEXT: s_endpgm
19991 ;
19992 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
19993 ; GFX11-WGP: ; %bb.0: ; %entry
19994 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
19995 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
19996 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
19997 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
19998 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
19999 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20000 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
20001 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
20002 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20003 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
20004 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20005 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20006 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20007 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20008 ; GFX11-WGP-NEXT: buffer_gl0_inv
20009 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20010 ; GFX11-WGP-NEXT: s_endpgm
20011 ;
20012 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
20013 ; GFX11-CU: ; %bb.0: ; %entry
20014 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20015 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
20016 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20017 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20018 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20019 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20020 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
20021 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
20022 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20023 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
20024 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20025 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20026 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20027 ; GFX11-CU-NEXT: s_endpgm
20028 ;
20029 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
20030 ; GFX12-WGP: ; %bb.0: ; %entry
20031 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20032 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
20033 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20034 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20035 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20036 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20037 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
20038 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
20039 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20040 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
20041 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20042 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20043 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20044 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20045 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20046 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20047 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20048 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20049 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20050 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20051 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20052 ; GFX12-WGP-NEXT: s_endpgm
20053 ;
20054 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
20055 ; GFX12-CU: ; %bb.0: ; %entry
20056 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20057 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
20058 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20059 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20060 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20061 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20062 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
20063 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
20064 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20065 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
20066 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
20067 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20068 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20069 ; GFX12-CU-NEXT: s_endpgm
20070 ptr addrspace(1) %out, i32 %in, i32 %old) {
20071 entry:
20072 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
20073 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
20074 %val0 = extractvalue { i32, i1 } %val, 0
20075 store i32 %val0, ptr addrspace(1) %out, align 4
20076 ret void
20077 }
20079 define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
20080 ; GFX6-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20081 ; GFX6: ; %bb.0: ; %entry
20082 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20083 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
20084 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
20085 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
20086 ; GFX6-NEXT: s_mov_b32 s12, s5
20087 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
20088 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
20089 ; GFX6-NEXT: s_mov_b32 s11, -1
20090 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
20091 ; GFX6-NEXT: s_mov_b32 s5, s12
20092 ; GFX6-NEXT: s_mov_b32 s6, s11
20093 ; GFX6-NEXT: s_mov_b32 s7, s10
20094 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
20095 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
20096 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20097 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
20098 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
20099 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20100 ; GFX6-NEXT: s_waitcnt vmcnt(0)
20101 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
20102 ; GFX6-NEXT: s_endpgm
20103 ;
20104 ; GFX7-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20105 ; GFX7: ; %bb.0: ; %entry
20106 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20107 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20108 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20109 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20110 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20111 ; GFX7-NEXT: s_mov_b32 s6, s4
20112 ; GFX7-NEXT: s_mov_b32 s7, s5
20113 ; GFX7-NEXT: s_mov_b32 s11, s12
20114 ; GFX7-NEXT: s_mov_b32 s10, s13
20115 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20116 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20117 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20118 ; GFX7-NEXT: s_mov_b32 s7, s10
20119 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20120 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20121 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20122 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20123 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20124 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20125 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20126 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20127 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20128 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20129 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20130 ; GFX7-NEXT: s_endpgm
20131 ;
20132 ; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20133 ; GFX10-WGP: ; %bb.0: ; %entry
20134 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
20135 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
20136 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20137 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
20138 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
20139 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20140 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20141 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
20142 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20143 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
20144 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20145 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20146 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20147 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20148 ; GFX10-WGP-NEXT: buffer_gl0_inv
20149 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
20150 ; GFX10-WGP-NEXT: s_endpgm
20151 ;
20152 ; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20153 ; GFX10-CU: ; %bb.0: ; %entry
20154 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
20155 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
20156 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20157 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
20158 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
20159 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20160 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20161 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
20162 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20163 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
20164 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20165 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20166 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
20167 ; GFX10-CU-NEXT: s_endpgm
20168 ;
20169 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20170 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20171 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20172 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20173 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20174 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20175 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
20176 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
20177 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
20178 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
20179 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
20180 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
20181 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
20182 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20183 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
20184 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
20185 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20186 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
20187 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
20188 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20189 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20190 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
20191 ; SKIP-CACHE-INV-NEXT: s_endpgm
20192 ;
20193 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20194 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20195 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20196 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20197 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20198 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20199 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20200 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20201 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20202 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20203 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20204 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20205 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20206 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20207 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20208 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20209 ;
20210 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20211 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20212 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20213 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20214 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20215 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20216 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20217 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20218 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20219 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20220 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20221 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20222 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20223 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20224 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20225 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20226 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20227 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20228 ;
20229 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20230 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20231 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20232 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20233 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20234 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20235 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20236 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20237 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20238 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20239 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20240 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20241 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20242 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20243 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20244 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20245 ;
20246 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20247 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20248 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20249 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20250 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20251 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20252 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20253 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20254 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20255 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20256 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20257 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20258 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20259 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20260 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20261 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20262 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20263 ; GFX940-TGSPLIT-NEXT: s_endpgm
20264 ;
20265 ; GFX11-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20266 ; GFX11-WGP: ; %bb.0: ; %entry
20267 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20268 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
20269 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20270 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20271 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20272 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20273 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
20274 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
20275 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20276 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
20277 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20278 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20279 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20280 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20281 ; GFX11-WGP-NEXT: buffer_gl0_inv
20282 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20283 ; GFX11-WGP-NEXT: s_endpgm
20284 ;
20285 ; GFX11-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20286 ; GFX11-CU: ; %bb.0: ; %entry
20287 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20288 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
20289 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20290 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20291 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20292 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20293 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
20294 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
20295 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20296 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
20297 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20298 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20299 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20300 ; GFX11-CU-NEXT: s_endpgm
20301 ;
20302 ; GFX12-WGP-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20303 ; GFX12-WGP: ; %bb.0: ; %entry
20304 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20305 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
20306 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20307 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20308 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20309 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20310 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
20311 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
20312 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20313 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
20314 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20315 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20316 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20317 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20318 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20319 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20320 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20321 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20322 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20323 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20324 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20325 ; GFX12-WGP-NEXT: s_endpgm
20326 ;
20327 ; GFX12-CU-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
20328 ; GFX12-CU: ; %bb.0: ; %entry
20329 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20330 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
20331 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20332 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20333 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20334 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20335 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
20336 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
20337 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20338 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
20339 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
20340 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20341 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20342 ; GFX12-CU-NEXT: s_endpgm
20343 ptr addrspace(1) %out, i32 %in, i32 %old) {
20344 entry:
20345 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
20346 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst
20347 %val0 = extractvalue { i32, i1 } %val, 0
20348 store i32 %val0, ptr addrspace(1) %out, align 4
20349 ret void
20350 }
20352 define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
20353 ; GFX6-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20354 ; GFX6: ; %bb.0: ; %entry
20355 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20356 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
20357 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
20358 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
20359 ; GFX6-NEXT: s_mov_b32 s12, s5
20360 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
20361 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
20362 ; GFX6-NEXT: s_mov_b32 s11, -1
20363 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
20364 ; GFX6-NEXT: s_mov_b32 s5, s12
20365 ; GFX6-NEXT: s_mov_b32 s6, s11
20366 ; GFX6-NEXT: s_mov_b32 s7, s10
20367 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
20368 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
20369 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20370 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
20371 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
20372 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20373 ; GFX6-NEXT: s_waitcnt vmcnt(0)
20374 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
20375 ; GFX6-NEXT: s_endpgm
20376 ;
20377 ; GFX7-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20378 ; GFX7: ; %bb.0: ; %entry
20379 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20380 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20381 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20382 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20383 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20384 ; GFX7-NEXT: s_mov_b32 s6, s4
20385 ; GFX7-NEXT: s_mov_b32 s7, s5
20386 ; GFX7-NEXT: s_mov_b32 s11, s12
20387 ; GFX7-NEXT: s_mov_b32 s10, s13
20388 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20389 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20390 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20391 ; GFX7-NEXT: s_mov_b32 s7, s10
20392 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20393 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20394 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20395 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20396 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20397 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20398 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20399 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20400 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20401 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20402 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20403 ; GFX7-NEXT: s_endpgm
20404 ;
20405 ; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20406 ; GFX10-WGP: ; %bb.0: ; %entry
20407 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
20408 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
20409 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20410 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
20411 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
20412 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20413 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20414 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
20415 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20416 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
20417 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20418 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20419 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20420 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20421 ; GFX10-WGP-NEXT: buffer_gl0_inv
20422 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
20423 ; GFX10-WGP-NEXT: s_endpgm
20424 ;
20425 ; GFX10-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20426 ; GFX10-CU: ; %bb.0: ; %entry
20427 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
20428 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
20429 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20430 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
20431 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
20432 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20433 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20434 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
20435 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20436 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
20437 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20438 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20439 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
20440 ; GFX10-CU-NEXT: s_endpgm
20441 ;
20442 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20443 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20444 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20445 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20446 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20447 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20448 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
20449 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
20450 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
20451 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
20452 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
20453 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
20454 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
20455 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20456 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
20457 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
20458 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20459 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
20460 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
20461 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20462 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20463 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
20464 ; SKIP-CACHE-INV-NEXT: s_endpgm
20465 ;
20466 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20467 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20468 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20469 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20470 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20471 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20472 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20473 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20474 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20475 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20476 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20477 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20478 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20479 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20480 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20481 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20482 ;
20483 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20484 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20485 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20486 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20487 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20488 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20489 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20490 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20491 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20492 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20493 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20494 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20495 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20496 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20497 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20498 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20499 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20500 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20501 ;
20502 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20503 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20504 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20505 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20506 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20507 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20508 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20509 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20510 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20511 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20512 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20513 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20514 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20515 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20516 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20517 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20518 ;
20519 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20520 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20521 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20522 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20523 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20524 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20525 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20526 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20527 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20528 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20529 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20530 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20531 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20532 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20533 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20534 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20535 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20536 ; GFX940-TGSPLIT-NEXT: s_endpgm
20537 ;
20538 ; GFX11-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20539 ; GFX11-WGP: ; %bb.0: ; %entry
20540 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20541 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
20542 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20543 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20544 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20545 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20546 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
20547 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
20548 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20549 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
20550 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20551 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20552 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20553 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20554 ; GFX11-WGP-NEXT: buffer_gl0_inv
20555 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20556 ; GFX11-WGP-NEXT: s_endpgm
20557 ;
20558 ; GFX11-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20559 ; GFX11-CU: ; %bb.0: ; %entry
20560 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20561 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
20562 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20563 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20564 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20565 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20566 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
20567 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
20568 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20569 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
20570 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20571 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20572 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20573 ; GFX11-CU-NEXT: s_endpgm
20574 ;
20575 ; GFX12-WGP-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20576 ; GFX12-WGP: ; %bb.0: ; %entry
20577 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20578 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
20579 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20580 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20581 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20582 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20583 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
20584 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
20585 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20586 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
20587 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20588 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20589 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20590 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20591 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20592 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20593 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20594 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20595 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20596 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20597 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20598 ; GFX12-WGP-NEXT: s_endpgm
20599 ;
20600 ; GFX12-CU-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
20601 ; GFX12-CU: ; %bb.0: ; %entry
20602 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20603 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
20604 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20605 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20606 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20607 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20608 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
20609 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
20610 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20611 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
20612 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
20613 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20614 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20615 ; GFX12-CU-NEXT: s_endpgm
20616 ptr addrspace(1) %out, i32 %in, i32 %old) {
20617 entry:
20618 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
20619 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst
20620 %val0 = extractvalue { i32, i1 } %val, 0
20621 store i32 %val0, ptr addrspace(1) %out, align 4
20622 ret void
20623 }
20625 define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_ret_cmpxchg(
20626 ; GFX6-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20627 ; GFX6: ; %bb.0: ; %entry
20628 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20629 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
20630 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
20631 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
20632 ; GFX6-NEXT: s_mov_b32 s12, s5
20633 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
20634 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
20635 ; GFX6-NEXT: s_mov_b32 s11, -1
20636 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
20637 ; GFX6-NEXT: s_mov_b32 s5, s12
20638 ; GFX6-NEXT: s_mov_b32 s6, s11
20639 ; GFX6-NEXT: s_mov_b32 s7, s10
20640 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
20641 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
20642 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20643 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
20644 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
20645 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20646 ; GFX6-NEXT: s_waitcnt vmcnt(0)
20647 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
20648 ; GFX6-NEXT: s_endpgm
20649 ;
20650 ; GFX7-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20651 ; GFX7: ; %bb.0: ; %entry
20652 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20653 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20654 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20655 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20656 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20657 ; GFX7-NEXT: s_mov_b32 s6, s4
20658 ; GFX7-NEXT: s_mov_b32 s7, s5
20659 ; GFX7-NEXT: s_mov_b32 s11, s12
20660 ; GFX7-NEXT: s_mov_b32 s10, s13
20661 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20662 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20663 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20664 ; GFX7-NEXT: s_mov_b32 s7, s10
20665 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20666 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20667 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20668 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20669 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20670 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20671 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20672 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20673 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20674 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20675 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20676 ; GFX7-NEXT: s_endpgm
20677 ;
20678 ; GFX10-WGP-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20679 ; GFX10-WGP: ; %bb.0: ; %entry
20680 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
20681 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
20682 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20683 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
20684 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
20685 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20686 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20687 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
20688 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20689 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
20690 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20691 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20692 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20693 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20694 ; GFX10-WGP-NEXT: buffer_gl0_inv
20695 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
20696 ; GFX10-WGP-NEXT: s_endpgm
20697 ;
20698 ; GFX10-CU-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20699 ; GFX10-CU: ; %bb.0: ; %entry
20700 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
20701 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
20702 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20703 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
20704 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
20705 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20706 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20707 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
20708 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20709 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
20710 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20711 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20712 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
20713 ; GFX10-CU-NEXT: s_endpgm
20714 ;
20715 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20716 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20717 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20718 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20719 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20720 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20721 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
20722 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
20723 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
20724 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
20725 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
20726 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
20727 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
20728 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
20729 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
20730 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
20731 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20732 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
20733 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
20734 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20735 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
20736 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
20737 ; SKIP-CACHE-INV-NEXT: s_endpgm
20738 ;
20739 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20740 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
20741 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20742 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20743 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20744 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20745 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20746 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20747 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20748 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20749 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20750 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20751 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20752 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20753 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20754 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
20755 ;
20756 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20757 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
20758 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
20759 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20760 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20761 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
20762 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
20763 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20764 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
20765 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
20766 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20767 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20768 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20769 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
20770 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20771 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
20772 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
20773 ; GFX90A-TGSPLIT-NEXT: s_endpgm
20774 ;
20775 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20776 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
20777 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20778 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20779 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20780 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20781 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20782 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20783 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20784 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20785 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20786 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20787 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20788 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
20789 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20790 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
20791 ;
20792 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20793 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
20794 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
20795 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
20796 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
20797 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
20798 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
20799 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
20800 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
20801 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
20802 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20803 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
20804 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20805 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
20806 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
20807 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
20808 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
20809 ; GFX940-TGSPLIT-NEXT: s_endpgm
20810 ;
20811 ; GFX11-WGP-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20812 ; GFX11-WGP: ; %bb.0: ; %entry
20813 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20814 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
20815 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20816 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20817 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20818 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
20819 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
20820 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
20821 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20822 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
20823 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20824 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20825 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20826 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
20827 ; GFX11-WGP-NEXT: buffer_gl0_inv
20828 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20829 ; GFX11-WGP-NEXT: s_endpgm
20830 ;
20831 ; GFX11-CU-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20832 ; GFX11-CU: ; %bb.0: ; %entry
20833 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20834 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
20835 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20836 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20837 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20838 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
20839 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
20840 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
20841 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20842 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
20843 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
20844 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
20845 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20846 ; GFX11-CU-NEXT: s_endpgm
20847 ;
20848 ; GFX12-WGP-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20849 ; GFX12-WGP: ; %bb.0: ; %entry
20850 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
20851 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
20852 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20853 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
20854 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
20855 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
20856 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
20857 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
20858 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20859 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
20860 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
20861 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20862 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20863 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20864 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
20865 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
20866 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
20867 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
20868 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
20869 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
20870 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
20871 ; GFX12-WGP-NEXT: s_endpgm
20872 ;
20873 ; GFX12-CU-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
20874 ; GFX12-CU: ; %bb.0: ; %entry
20875 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
20876 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
20877 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
20878 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
20879 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
20880 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
20881 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
20882 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
20883 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20884 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
20885 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
20886 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
20887 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
20888 ; GFX12-CU-NEXT: s_endpgm
20889 ptr addrspace(1) %out, i32 %in, i32 %old) {
20890 entry:
20891 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
20892 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst
20893 %val0 = extractvalue { i32, i1 } %val, 0
20894 store i32 %val0, ptr addrspace(1) %out, align 4
20895 ret void
20896 }
20898 define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
20899 ; GFX6-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20900 ; GFX6: ; %bb.0: ; %entry
20901 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20902 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
20903 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
20904 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
20905 ; GFX6-NEXT: s_mov_b32 s12, s5
20906 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
20907 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
20908 ; GFX6-NEXT: s_mov_b32 s11, -1
20909 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
20910 ; GFX6-NEXT: s_mov_b32 s5, s12
20911 ; GFX6-NEXT: s_mov_b32 s6, s11
20912 ; GFX6-NEXT: s_mov_b32 s7, s10
20913 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
20914 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
20915 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
20916 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
20917 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
20918 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
20919 ; GFX6-NEXT: s_waitcnt vmcnt(0)
20920 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
20921 ; GFX6-NEXT: s_endpgm
20922 ;
20923 ; GFX7-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20924 ; GFX7: ; %bb.0: ; %entry
20925 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
20926 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
20927 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
20928 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
20929 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
20930 ; GFX7-NEXT: s_mov_b32 s6, s4
20931 ; GFX7-NEXT: s_mov_b32 s7, s5
20932 ; GFX7-NEXT: s_mov_b32 s11, s12
20933 ; GFX7-NEXT: s_mov_b32 s10, s13
20934 ; GFX7-NEXT: s_add_u32 s6, s6, s11
20935 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
20936 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
20937 ; GFX7-NEXT: s_mov_b32 s7, s10
20938 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
20939 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
20940 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
20941 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
20942 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
20943 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
20944 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
20945 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
20946 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
20947 ; GFX7-NEXT: s_waitcnt vmcnt(0)
20948 ; GFX7-NEXT: flat_store_dword v[0:1], v2
20949 ; GFX7-NEXT: s_endpgm
20950 ;
20951 ; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20952 ; GFX10-WGP: ; %bb.0: ; %entry
20953 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
20954 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
20955 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20956 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
20957 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
20958 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
20959 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
20960 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
20961 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20962 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
20963 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20964 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
20965 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20966 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
20967 ; GFX10-WGP-NEXT: buffer_gl0_inv
20968 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
20969 ; GFX10-WGP-NEXT: s_endpgm
20970 ;
20971 ; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20972 ; GFX10-CU: ; %bb.0: ; %entry
20973 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
20974 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
20975 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
20976 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
20977 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
20978 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
20979 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
20980 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
20981 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
20982 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
20983 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
20984 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
20985 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
20986 ; GFX10-CU-NEXT: s_endpgm
20987 ;
20988 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
20989 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
20990 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
20991 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
20992 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
20993 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
20994 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
20995 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
20996 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
20997 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
20998 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
20999 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
21000 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
21001 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
21002 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
21003 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
21004 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
21005 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
21006 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
21007 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
21008 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21009 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
21010 ; SKIP-CACHE-INV-NEXT: s_endpgm
21011 ;
21012 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21013 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
21014 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
21015 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21016 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21017 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21018 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21019 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21020 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21021 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
21022 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21023 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21024 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
21025 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21026 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
21027 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21028 ;
21029 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21030 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21031 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
21032 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21033 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21034 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21035 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21036 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21037 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21038 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
21039 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21040 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21041 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21042 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
21043 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21044 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21045 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
21046 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21047 ;
21048 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21049 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21050 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
21051 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21052 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21053 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21054 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21055 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21056 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21057 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
21058 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21059 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21060 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
21061 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21062 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
21063 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
21064 ;
21065 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21066 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
21067 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
21068 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21069 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21070 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21071 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21072 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21073 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21074 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
21075 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21076 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21077 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21078 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
21079 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21080 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
21081 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
21082 ; GFX940-TGSPLIT-NEXT: s_endpgm
21083 ;
21084 ; GFX11-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21085 ; GFX11-WGP: ; %bb.0: ; %entry
21086 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
21087 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
21088 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21089 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21090 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21091 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21092 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
21093 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
21094 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21095 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
21096 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21097 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21098 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
21099 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21100 ; GFX11-WGP-NEXT: buffer_gl0_inv
21101 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
21102 ; GFX11-WGP-NEXT: s_endpgm
21103 ;
21104 ; GFX11-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21105 ; GFX11-CU: ; %bb.0: ; %entry
21106 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
21107 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
21108 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21109 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21110 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21111 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21112 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
21113 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
21114 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21115 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
21116 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
21117 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21118 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
21119 ; GFX11-CU-NEXT: s_endpgm
21120 ;
21121 ; GFX12-WGP-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21122 ; GFX12-WGP: ; %bb.0: ; %entry
21123 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
21124 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
21125 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21126 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21127 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21128 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21129 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
21130 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
21131 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21132 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
21133 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
21134 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21135 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21136 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21137 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21138 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
21139 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21140 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21141 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21142 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
21143 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
21144 ; GFX12-WGP-NEXT: s_endpgm
21145 ;
21146 ; GFX12-CU-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
21147 ; GFX12-CU: ; %bb.0: ; %entry
21148 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
21149 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
21150 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21151 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21152 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21153 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21154 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
21155 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
21156 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21157 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
21158 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
21159 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21160 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
21161 ; GFX12-CU-NEXT: s_endpgm
21162 ptr addrspace(1) %out, i32 %in, i32 %old) {
21163 entry:
21164 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
21165 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst
21166 %val0 = extractvalue { i32, i1 } %val, 0
21167 store i32 %val0, ptr addrspace(1) %out, align 4
21168 ret void
21169 }
21171 define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
21172 ; GFX6-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21173 ; GFX6: ; %bb.0: ; %entry
21174 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21175 ; GFX6-NEXT: s_load_dword s9, s[6:7], 0x2
21176 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x3
21177 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
21178 ; GFX6-NEXT: s_mov_b32 s12, s5
21179 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
21180 ; GFX6-NEXT: s_mov_b32 s10, 0x100f000
21181 ; GFX6-NEXT: s_mov_b32 s11, -1
21182 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
21183 ; GFX6-NEXT: s_mov_b32 s5, s12
21184 ; GFX6-NEXT: s_mov_b32 s6, s11
21185 ; GFX6-NEXT: s_mov_b32 s7, s10
21186 ; GFX6-NEXT: v_mov_b32_e32 v0, s9
21187 ; GFX6-NEXT: v_mov_b32_e32 v2, s8
21188 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
21189 ; GFX6-NEXT: v_mov_b32_e32 v1, v2
21190 ; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
21191 ; GFX6-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
21192 ; GFX6-NEXT: s_waitcnt vmcnt(0)
21193 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
21194 ; GFX6-NEXT: s_endpgm
21195 ;
21196 ; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21197 ; GFX7: ; %bb.0: ; %entry
21198 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
21199 ; GFX7-NEXT: s_load_dword s9, s[6:7], 0x2
21200 ; GFX7-NEXT: s_load_dword s8, s[6:7], 0x3
21201 ; GFX7-NEXT: s_mov_b64 s[12:13], 16
21202 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21203 ; GFX7-NEXT: s_mov_b32 s6, s4
21204 ; GFX7-NEXT: s_mov_b32 s7, s5
21205 ; GFX7-NEXT: s_mov_b32 s11, s12
21206 ; GFX7-NEXT: s_mov_b32 s10, s13
21207 ; GFX7-NEXT: s_add_u32 s6, s6, s11
21208 ; GFX7-NEXT: s_addc_u32 s10, s7, s10
21209 ; GFX7-NEXT: ; kill: def $sgpr6 killed $sgpr6 def $sgpr6_sgpr7
21210 ; GFX7-NEXT: s_mov_b32 s7, s10
21211 ; GFX7-NEXT: v_mov_b32_e32 v2, s9
21212 ; GFX7-NEXT: v_mov_b32_e32 v0, s8
21213 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21214 ; GFX7-NEXT: v_mov_b32_e32 v3, v0
21215 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
21216 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
21217 ; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
21218 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
21219 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
21220 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21221 ; GFX7-NEXT: flat_store_dword v[0:1], v2
21222 ; GFX7-NEXT: s_endpgm
21223 ;
21224 ; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21225 ; GFX10-WGP: ; %bb.0: ; %entry
21226 ; GFX10-WGP-NEXT: s_mov_b64 s[8:9], s[6:7]
21227 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
21228 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21229 ; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x8
21230 ; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0xc
21231 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
21232 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
21233 ; GFX10-WGP-NEXT: v_mov_b32_e32 v3, s6
21234 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21235 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v3
21236 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21237 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21238 ; GFX10-WGP-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
21239 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
21240 ; GFX10-WGP-NEXT: buffer_gl0_inv
21241 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
21242 ; GFX10-WGP-NEXT: s_endpgm
21243 ;
21244 ; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21245 ; GFX10-CU: ; %bb.0: ; %entry
21246 ; GFX10-CU-NEXT: s_mov_b64 s[8:9], s[6:7]
21247 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
21248 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21249 ; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x8
21250 ; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0xc
21251 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
21252 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
21253 ; GFX10-CU-NEXT: v_mov_b32_e32 v3, s6
21254 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21255 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v3
21256 ; GFX10-CU-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[4:5] offset:16 glc
21257 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
21258 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
21259 ; GFX10-CU-NEXT: s_endpgm
21260 ;
21261 ; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21262 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
21263 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
21264 ; SKIP-CACHE-INV-NEXT: s_load_dword s5, s[2:3], 0x2
21265 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x3
21266 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
21267 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s8, s1
21268 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
21269 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, 0xf000
21270 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, -1
21271 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
21272 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s8
21273 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s7
21274 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s6
21275 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s5
21276 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s4
21277 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
21278 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
21279 ; SKIP-CACHE-INV-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
21280 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
21281 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
21282 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
21283 ; SKIP-CACHE-INV-NEXT: s_endpgm
21284 ;
21285 ; GFX90A-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21286 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
21287 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
21288 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21289 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21290 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21291 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21292 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21293 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21294 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
21295 ; GFX90A-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21296 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21297 ; GFX90A-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
21298 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21299 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
21300 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
21301 ;
21302 ; GFX90A-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21303 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
21304 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[8:9], s[6:7]
21305 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21306 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
21307 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x8
21308 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0xc
21309 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21310 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
21311 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
21312 ; GFX90A-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21313 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21314 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21315 ; GFX90A-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[4:5] offset:16 glc
21316 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21317 ; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol
21318 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
21319 ; GFX90A-TGSPLIT-NEXT: s_endpgm
21320 ;
21321 ; GFX940-NOTTGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21322 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
21323 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
21324 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21325 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21326 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21327 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21328 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21329 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21330 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
21331 ; GFX940-NOTTGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21332 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21333 ; GFX940-NOTTGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
21334 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
21335 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
21336 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
21337 ;
21338 ; GFX940-TGSPLIT-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21339 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
21340 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[2:3]
21341 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
21342 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
21343 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x8
21344 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0xc
21345 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
21346 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
21347 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
21348 ; GFX940-TGSPLIT-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
21349 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v3, v1
21350 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21351 ; GFX940-TGSPLIT-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[0:1] offset:16 sc0
21352 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
21353 ; GFX940-TGSPLIT-NEXT: buffer_inv sc0
21354 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
21355 ; GFX940-TGSPLIT-NEXT: s_endpgm
21356 ;
21357 ; GFX11-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21358 ; GFX11-WGP: ; %bb.0: ; %entry
21359 ; GFX11-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
21360 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
21361 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21362 ; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21363 ; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21364 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
21365 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
21366 ; GFX11-WGP-NEXT: v_mov_b32_e32 v3, s2
21367 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21368 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v3
21369 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21370 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
21371 ; GFX11-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
21372 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
21373 ; GFX11-WGP-NEXT: buffer_gl0_inv
21374 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
21375 ; GFX11-WGP-NEXT: s_endpgm
21376 ;
21377 ; GFX11-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21378 ; GFX11-CU: ; %bb.0: ; %entry
21379 ; GFX11-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
21380 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
21381 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21382 ; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21383 ; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21384 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
21385 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
21386 ; GFX11-CU-NEXT: v_mov_b32_e32 v3, s2
21387 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21388 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v3
21389 ; GFX11-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 glc
21390 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
21391 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
21392 ; GFX11-CU-NEXT: s_endpgm
21393 ;
21394 ; GFX12-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21395 ; GFX12-WGP: ; %bb.0: ; %entry
21396 ; GFX12-WGP-NEXT: s_mov_b64 s[4:5], s[2:3]
21397 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
21398 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21399 ; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x8
21400 ; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0xc
21401 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
21402 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
21403 ; GFX12-WGP-NEXT: v_mov_b32_e32 v3, s2
21404 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21405 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v3
21406 ; GFX12-WGP-NEXT: global_wb scope:SCOPE_SE
21407 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21408 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21409 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21410 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
21411 ; GFX12-WGP-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SE
21412 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
21413 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
21414 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
21415 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
21416 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
21417 ; GFX12-WGP-NEXT: s_endpgm
21418 ;
21419 ; GFX12-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
21420 ; GFX12-CU: ; %bb.0: ; %entry
21421 ; GFX12-CU-NEXT: s_mov_b64 s[4:5], s[2:3]
21422 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
21423 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
21424 ; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x8
21425 ; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0xc
21426 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
21427 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
21428 ; GFX12-CU-NEXT: v_mov_b32_e32 v3, s2
21429 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
21430 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v3
21431 ; GFX12-CU-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:16 th:TH_ATOMIC_RETURN
21432 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
21433 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
21434 ; GFX12-CU-NEXT: s_endpgm
21435 ptr addrspace(1) %out, i32 %in, i32 %old) {
21436 entry:
21437 %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
21438 %val = cmpxchg volatile ptr addrspace(1) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
21439 %val0 = extractvalue { i32, i1 } %val, 0
21440 store i32 %val0, ptr addrspace(1) %out, align 4
21441 ret void
21442 }