| blob | f10715033e4338428bfb747a69f60925d699ed64 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
11 define amdgpu_kernel void @flat_nontemporal_load_0(
12 ; GFX7-LABEL: flat_nontemporal_load_0:
13 ; GFX7: ; %bb.0: ; %entry
14 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
15 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
16 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
17 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
18 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
19 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
20 ; GFX7-NEXT: s_waitcnt vmcnt(0)
21 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
22 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
23 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
24 ; GFX7-NEXT: flat_store_dword v[0:1], v2
25 ; GFX7-NEXT: s_endpgm
26 ;
27 ; GFX10-WGP-LABEL: flat_nontemporal_load_0:
28 ; GFX10-WGP: ; %bb.0: ; %entry
29 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
30 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
31 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
32 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
33 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
34 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
35 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
36 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
37 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
38 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
39 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
40 ; GFX10-WGP-NEXT: s_endpgm
41 ;
42 ; GFX10-CU-LABEL: flat_nontemporal_load_0:
43 ; GFX10-CU: ; %bb.0: ; %entry
44 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
45 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
46 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
47 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
48 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
49 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
50 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
51 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
52 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
53 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
54 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
55 ; GFX10-CU-NEXT: s_endpgm
56 ;
57 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_0:
58 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
59 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
60 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
61 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
62 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
63 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
64 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
65 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
66 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
67 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
68 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
69 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
70 ; SKIP-CACHE-INV-NEXT: s_endpgm
71 ;
72 ; GFX11-WGP-LABEL: flat_nontemporal_load_0:
73 ; GFX11-WGP: ; %bb.0: ; %entry
74 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
75 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
76 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
77 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
78 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
79 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
80 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
81 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
82 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
83 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
84 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
85 ; GFX11-WGP-NEXT: s_endpgm
86 ;
87 ; GFX11-CU-LABEL: flat_nontemporal_load_0:
88 ; GFX11-CU: ; %bb.0: ; %entry
89 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
90 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
91 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
92 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
93 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
94 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
95 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
96 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
97 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
98 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
99 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
100 ; GFX11-CU-NEXT: s_endpgm
101 ;
102 ; GFX12-WGP-LABEL: flat_nontemporal_load_0:
103 ; GFX12-WGP: ; %bb.0: ; %entry
104 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
105 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
106 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
107 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
108 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
109 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
110 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
111 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
112 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
113 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
114 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
115 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
116 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
117 ; GFX12-WGP-NEXT: s_endpgm
118 ;
119 ; GFX12-CU-LABEL: flat_nontemporal_load_0:
120 ; GFX12-CU: ; %bb.0: ; %entry
121 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
122 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
123 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
124 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
125 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
126 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
127 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
128 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
129 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
130 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
131 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
132 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
133 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
134 ; GFX12-CU-NEXT: s_endpgm
135 ptr %in, ptr %out) {
136 entry:
137 %val = load volatile i32, ptr %in, align 4
138 store i32 %val, ptr %out
139 ret void
140 }
142 define amdgpu_kernel void @flat_nontemporal_load_1(
143 ; GFX7-LABEL: flat_nontemporal_load_1:
144 ; GFX7: ; %bb.0: ; %entry
145 ; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]
146 ; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
147 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
148 ; GFX7-NEXT: s_mov_b32 s6, 2
149 ; GFX7-NEXT: v_lshlrev_b32_e64 v1, s6, v0
150 ; GFX7-NEXT: s_mov_b32 s6, 0
151 ; GFX7-NEXT: ; implicit-def: $sgpr6
152 ; GFX7-NEXT: v_mov_b32_e32 v0, 0
153 ; GFX7-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
154 ; GFX7-NEXT: v_mov_b32_e32 v2, v0
155 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
156 ; GFX7-NEXT: s_mov_b32 s6, s8
157 ; GFX7-NEXT: v_mov_b32_e32 v0, v1
158 ; GFX7-NEXT: s_mov_b32 s8, s9
159 ; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec
160 ; GFX7-NEXT: v_add_i32_e64 v0, s[6:7], s6, v0
161 ; GFX7-NEXT: v_mov_b32_e32 v1, s8
162 ; GFX7-NEXT: v_addc_u32_e64 v2, s[6:7], v1, v2, s[6:7]
163 ; GFX7-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
164 ; GFX7-NEXT: v_mov_b32_e32 v1, v2
165 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
166 ; GFX7-NEXT: s_waitcnt vmcnt(0)
167 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
168 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
169 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
170 ; GFX7-NEXT: flat_store_dword v[0:1], v2
171 ; GFX7-NEXT: s_endpgm
172 ;
173 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
174 ; GFX10-WGP: ; %bb.0: ; %entry
175 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]
176 ; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
177 ; GFX10-WGP-NEXT: s_nop 0
178 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
179 ; GFX10-WGP-NEXT: s_mov_b32 s6, 2
180 ; GFX10-WGP-NEXT: v_lshlrev_b32_e64 v1, s6, v0
181 ; GFX10-WGP-NEXT: s_mov_b32 s6, 0
182 ; GFX10-WGP-NEXT: ; implicit-def: $sgpr6
183 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
184 ; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
185 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v0
186 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
187 ; GFX10-WGP-NEXT: s_mov_b32 s7, s8
188 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v1
189 ; GFX10-WGP-NEXT: s_mov_b32 s6, s9
190 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
191 ; GFX10-WGP-NEXT: v_add_co_u32 v0, s7, s7, v0
192 ; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v2, s6, s6, v1, s7
193 ; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
194 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2
195 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
196 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
197 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
198 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
199 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
200 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
201 ; GFX10-WGP-NEXT: s_endpgm
202 ;
203 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
204 ; GFX10-CU: ; %bb.0: ; %entry
205 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]
206 ; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
207 ; GFX10-CU-NEXT: s_nop 0
208 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
209 ; GFX10-CU-NEXT: s_mov_b32 s6, 2
210 ; GFX10-CU-NEXT: v_lshlrev_b32_e64 v1, s6, v0
211 ; GFX10-CU-NEXT: s_mov_b32 s6, 0
212 ; GFX10-CU-NEXT: ; implicit-def: $sgpr6
213 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
214 ; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
215 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, v0
216 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
217 ; GFX10-CU-NEXT: s_mov_b32 s7, s8
218 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, v1
219 ; GFX10-CU-NEXT: s_mov_b32 s6, s9
220 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
221 ; GFX10-CU-NEXT: v_add_co_u32 v0, s7, s7, v0
222 ; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v2, s6, s6, v1, s7
223 ; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
224 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2
225 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
226 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
227 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
228 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
229 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
230 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
231 ; GFX10-CU-NEXT: s_endpgm
232 ;
233 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:
234 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
235 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]
236 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
237 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2
238 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, 2
239 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v1, s2, v0
240 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, 0
241 ; SKIP-CACHE-INV-NEXT: ; implicit-def: $sgpr2
242 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, 0
243 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
244 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, v0
245 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
246 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s4
247 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, v1
248 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s5
249 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec
250 ; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v0, s[2:3], s2, v0
251 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s4
252 ; SKIP-CACHE-INV-NEXT: v_addc_u32_e64 v2, s[2:3], v1, v2, s[2:3]
253 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
254 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2
255 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
256 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
257 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
258 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
259 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
260 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
261 ; SKIP-CACHE-INV-NEXT: s_endpgm
262 ;
263 ; GFX11-WGP-LABEL: flat_nontemporal_load_1:
264 ; GFX11-WGP: ; %bb.0: ; %entry
265 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
266 ; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
267 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
268 ; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
269 ; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2
270 ; GFX11-WGP-NEXT: s_mov_b32 s2, 2
271 ; GFX11-WGP-NEXT: v_lshlrev_b32_e64 v1, s2, v0
272 ; GFX11-WGP-NEXT: s_mov_b32 s2, 0
273 ; GFX11-WGP-NEXT: ; implicit-def: $sgpr2
274 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
275 ; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
276 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v0
277 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
278 ; GFX11-WGP-NEXT: s_mov_b32 s3, s4
279 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v1
280 ; GFX11-WGP-NEXT: s_mov_b32 s2, s5
281 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
282 ; GFX11-WGP-NEXT: v_add_co_u32 v0, s3, s3, v0
283 ; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
284 ; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
285 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2
286 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
287 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
288 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
289 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
290 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
291 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
292 ; GFX11-WGP-NEXT: s_endpgm
293 ;
294 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
295 ; GFX11-CU: ; %bb.0: ; %entry
296 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
297 ; GFX11-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
298 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
299 ; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
300 ; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2
301 ; GFX11-CU-NEXT: s_mov_b32 s2, 2
302 ; GFX11-CU-NEXT: v_lshlrev_b32_e64 v1, s2, v0
303 ; GFX11-CU-NEXT: s_mov_b32 s2, 0
304 ; GFX11-CU-NEXT: ; implicit-def: $sgpr2
305 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
306 ; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
307 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, v0
308 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
309 ; GFX11-CU-NEXT: s_mov_b32 s3, s4
310 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, v1
311 ; GFX11-CU-NEXT: s_mov_b32 s2, s5
312 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
313 ; GFX11-CU-NEXT: v_add_co_u32 v0, s3, s3, v0
314 ; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
315 ; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
316 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2
317 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
318 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
319 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
320 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
321 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
322 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
323 ; GFX11-CU-NEXT: s_endpgm
324 ;
325 ; GFX12-WGP-LABEL: flat_nontemporal_load_1:
326 ; GFX12-WGP: ; %bb.0: ; %entry
327 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]
328 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
329 ; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
330 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
331 ; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
332 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
333 ; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s2
334 ; GFX12-WGP-NEXT: s_mov_b32 s2, 2
335 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
336 ; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v1, s2, v0
337 ; GFX12-WGP-NEXT: s_mov_b32 s2, 0
338 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
339 ; GFX12-WGP-NEXT: ; implicit-def: $sgpr2
340 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
341 ; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
342 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v0
343 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
344 ; GFX12-WGP-NEXT: s_mov_b32 s3, s4
345 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
346 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v1
347 ; GFX12-WGP-NEXT: s_mov_b32 s2, s5
348 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
349 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
350 ; GFX12-WGP-NEXT: v_add_co_u32 v0, s3, s3, v0
351 ; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
352 ; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
353 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2
354 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
355 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
356 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
357 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
358 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
359 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
360 ; GFX12-WGP-NEXT: s_wait_dscnt 0x0
361 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
362 ; GFX12-WGP-NEXT: s_endpgm
363 ;
364 ; GFX12-CU-LABEL: flat_nontemporal_load_1:
365 ; GFX12-CU: ; %bb.0: ; %entry
366 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[4:5]
367 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
368 ; GFX12-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
369 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
370 ; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
371 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
372 ; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s2
373 ; GFX12-CU-NEXT: s_mov_b32 s2, 2
374 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
375 ; GFX12-CU-NEXT: v_lshlrev_b32_e64 v1, s2, v0
376 ; GFX12-CU-NEXT: s_mov_b32 s2, 0
377 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
378 ; GFX12-CU-NEXT: ; implicit-def: $sgpr2
379 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
380 ; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
381 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, v0
382 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
383 ; GFX12-CU-NEXT: s_mov_b32 s3, s4
384 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
385 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, v1
386 ; GFX12-CU-NEXT: s_mov_b32 s2, s5
387 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
388 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
389 ; GFX12-CU-NEXT: v_add_co_u32 v0, s3, s3, v0
390 ; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3
391 ; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
392 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2
393 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS
394 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
395 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
396 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
397 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
398 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
399 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
400 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
401 ; GFX12-CU-NEXT: s_endpgm
402 ptr %in, ptr %out) {
403 entry:
404 %tid = call i32 @llvm.amdgcn.workitem.id.x()
405 %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid
406 %val = load volatile i32, ptr %val.gep, align 4
407 store i32 %val, ptr %out
408 ret void
409 }
411 define amdgpu_kernel void @flat_nontemporal_store_0(
412 ; GFX7-LABEL: flat_nontemporal_store_0:
413 ; GFX7: ; %bb.0: ; %entry
414 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
415 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
416 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
417 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
418 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
419 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
420 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
421 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
422 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
423 ; GFX7-NEXT: flat_store_dword v[0:1], v2
424 ; GFX7-NEXT: s_waitcnt vmcnt(0)
425 ; GFX7-NEXT: s_endpgm
426 ;
427 ; GFX10-WGP-LABEL: flat_nontemporal_store_0:
428 ; GFX10-WGP: ; %bb.0: ; %entry
429 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
430 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
431 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
432 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
433 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
434 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
435 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
436 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
437 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
438 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
439 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
440 ; GFX10-WGP-NEXT: s_endpgm
441 ;
442 ; GFX10-CU-LABEL: flat_nontemporal_store_0:
443 ; GFX10-CU: ; %bb.0: ; %entry
444 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
445 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
446 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
447 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
448 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
449 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
450 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
451 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
452 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
453 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
454 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
455 ; GFX10-CU-NEXT: s_endpgm
456 ;
457 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_0:
458 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
459 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
460 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
461 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
462 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
463 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
464 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
465 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
466 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
467 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
468 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
469 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
470 ; SKIP-CACHE-INV-NEXT: s_endpgm
471 ;
472 ; GFX11-WGP-LABEL: flat_nontemporal_store_0:
473 ; GFX11-WGP: ; %bb.0: ; %entry
474 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
475 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
476 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
477 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
478 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
479 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
480 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
481 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
482 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
483 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc
484 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
485 ; GFX11-WGP-NEXT: s_endpgm
486 ;
487 ; GFX11-CU-LABEL: flat_nontemporal_store_0:
488 ; GFX11-CU: ; %bb.0: ; %entry
489 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
490 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
491 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
492 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
493 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
494 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
495 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
496 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
497 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
498 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
499 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
500 ; GFX11-CU-NEXT: s_endpgm
501 ;
502 ; GFX12-WGP-LABEL: flat_nontemporal_store_0:
503 ; GFX12-WGP: ; %bb.0: ; %entry
504 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
505 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
506 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
507 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
508 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
509 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]
510 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
511 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
512 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
513 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
514 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
515 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
516 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
517 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
518 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
519 ; GFX12-WGP-NEXT: s_endpgm
520 ;
521 ; GFX12-CU-LABEL: flat_nontemporal_store_0:
522 ; GFX12-CU: ; %bb.0: ; %entry
523 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
524 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
525 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
526 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
527 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
528 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
529 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
530 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
531 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
532 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
533 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
534 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
535 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
536 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
537 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
538 ; GFX12-CU-NEXT: s_endpgm
539 ptr %in, ptr %out) {
540 entry:
541 %val = load i32, ptr %in, align 4
542 store volatile i32 %val, ptr %out
543 ret void
544 }
546 define amdgpu_kernel void @flat_nontemporal_store_1(
547 ; GFX7-LABEL: flat_nontemporal_store_1:
548 ; GFX7: ; %bb.0: ; %entry
549 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
550 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
551 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
552 ; GFX7-NEXT: v_mov_b32_e32 v1, s4
553 ; GFX7-NEXT: v_mov_b32_e32 v2, s5
554 ; GFX7-NEXT: flat_load_dword v2, v[1:2]
555 ; GFX7-NEXT: s_mov_b32 s4, 2
556 ; GFX7-NEXT: v_lshlrev_b32_e64 v3, s4, v0
557 ; GFX7-NEXT: s_mov_b32 s4, 0
558 ; GFX7-NEXT: ; implicit-def: $sgpr4
559 ; GFX7-NEXT: v_mov_b32_e32 v0, 0
560 ; GFX7-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
561 ; GFX7-NEXT: v_mov_b32_e32 v4, v0
562 ; GFX7-NEXT: s_mov_b32 s4, s6
563 ; GFX7-NEXT: v_mov_b32_e32 v0, v3
564 ; GFX7-NEXT: s_mov_b32 s6, s7
565 ; GFX7-NEXT: v_mov_b32_e32 v3, v4
566 ; GFX7-NEXT: v_add_i32_e64 v0, s[4:5], s4, v0
567 ; GFX7-NEXT: v_mov_b32_e32 v1, s6
568 ; GFX7-NEXT: v_addc_u32_e64 v3, s[4:5], v1, v3, s[4:5]
569 ; GFX7-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
570 ; GFX7-NEXT: v_mov_b32_e32 v1, v3
571 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
572 ; GFX7-NEXT: flat_store_dword v[0:1], v2
573 ; GFX7-NEXT: s_waitcnt vmcnt(0)
574 ; GFX7-NEXT: s_endpgm
575 ;
576 ; GFX10-WGP-LABEL: flat_nontemporal_store_1:
577 ; GFX10-WGP: ; %bb.0: ; %entry
578 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
579 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
580 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
581 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4
582 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s5
583 ; GFX10-WGP-NEXT: flat_load_dword v2, v[1:2]
584 ; GFX10-WGP-NEXT: s_mov_b32 s4, 2
585 ; GFX10-WGP-NEXT: v_lshlrev_b32_e64 v3, s4, v0
586 ; GFX10-WGP-NEXT: s_mov_b32 s4, 0
587 ; GFX10-WGP-NEXT: ; implicit-def: $sgpr4
588 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
589 ; GFX10-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
590 ; GFX10-WGP-NEXT: v_mov_b32_e32 v4, v0
591 ; GFX10-WGP-NEXT: s_mov_b32 s5, s6
592 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v3
593 ; GFX10-WGP-NEXT: s_mov_b32 s4, s7
594 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v4
595 ; GFX10-WGP-NEXT: v_add_co_u32 v0, s5, s5, v0
596 ; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v3, s4, s4, v1, s5
597 ; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
598 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v3
599 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
600 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
601 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
602 ; GFX10-WGP-NEXT: s_endpgm
603 ;
604 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
605 ; GFX10-CU: ; %bb.0: ; %entry
606 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
607 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
608 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
609 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4
610 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s5
611 ; GFX10-CU-NEXT: flat_load_dword v2, v[1:2]
612 ; GFX10-CU-NEXT: s_mov_b32 s4, 2
613 ; GFX10-CU-NEXT: v_lshlrev_b32_e64 v3, s4, v0
614 ; GFX10-CU-NEXT: s_mov_b32 s4, 0
615 ; GFX10-CU-NEXT: ; implicit-def: $sgpr4
616 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
617 ; GFX10-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
618 ; GFX10-CU-NEXT: v_mov_b32_e32 v4, v0
619 ; GFX10-CU-NEXT: s_mov_b32 s5, s6
620 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, v3
621 ; GFX10-CU-NEXT: s_mov_b32 s4, s7
622 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, v4
623 ; GFX10-CU-NEXT: v_add_co_u32 v0, s5, s5, v0
624 ; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v3, s4, s4, v1, s5
625 ; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
626 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, v3
627 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
628 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
629 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
630 ; GFX10-CU-NEXT: s_endpgm
631 ;
632 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
633 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
634 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
635 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
636 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
637 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s1
638 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0
639 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[1:2]
640 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, 2
641 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v3, s0, v0
642 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, 0
643 ; SKIP-CACHE-INV-NEXT: ; implicit-def: $sgpr0
644 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, 0
645 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
646 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v4, v0
647 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s2
648 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, v3
649 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s3
650 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v4
651 ; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v0, s[0:1], s0, v0
652 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s2
653 ; SKIP-CACHE-INV-NEXT: v_addc_u32_e64 v3, s[0:1], v1, v3, s[0:1]
654 ; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
655 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v3
656 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
657 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
658 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
659 ; SKIP-CACHE-INV-NEXT: s_endpgm
660 ;
661 ; GFX11-WGP-LABEL: flat_nontemporal_store_1:
662 ; GFX11-WGP: ; %bb.0: ; %entry
663 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
664 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
665 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
666 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s1
667 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s0
668 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[1:2]
669 ; GFX11-WGP-NEXT: s_mov_b32 s0, 0x3ff
670 ; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s0
671 ; GFX11-WGP-NEXT: s_mov_b32 s0, 2
672 ; GFX11-WGP-NEXT: v_lshlrev_b32_e64 v3, s0, v0
673 ; GFX11-WGP-NEXT: s_mov_b32 s0, 0
674 ; GFX11-WGP-NEXT: ; implicit-def: $sgpr0
675 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
676 ; GFX11-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
677 ; GFX11-WGP-NEXT: v_mov_b32_e32 v4, v0
678 ; GFX11-WGP-NEXT: s_mov_b32 s1, s2
679 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v3
680 ; GFX11-WGP-NEXT: s_mov_b32 s0, s3
681 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v4
682 ; GFX11-WGP-NEXT: v_add_co_u32 v0, s1, s1, v0
683 ; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
684 ; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
685 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v3
686 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
687 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc
688 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
689 ; GFX11-WGP-NEXT: s_endpgm
690 ;
691 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
692 ; GFX11-CU: ; %bb.0: ; %entry
693 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
694 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
695 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
696 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s1
697 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s0
698 ; GFX11-CU-NEXT: flat_load_b32 v2, v[1:2]
699 ; GFX11-CU-NEXT: s_mov_b32 s0, 0x3ff
700 ; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s0
701 ; GFX11-CU-NEXT: s_mov_b32 s0, 2
702 ; GFX11-CU-NEXT: v_lshlrev_b32_e64 v3, s0, v0
703 ; GFX11-CU-NEXT: s_mov_b32 s0, 0
704 ; GFX11-CU-NEXT: ; implicit-def: $sgpr0
705 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
706 ; GFX11-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
707 ; GFX11-CU-NEXT: v_mov_b32_e32 v4, v0
708 ; GFX11-CU-NEXT: s_mov_b32 s1, s2
709 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, v3
710 ; GFX11-CU-NEXT: s_mov_b32 s0, s3
711 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, v4
712 ; GFX11-CU-NEXT: v_add_co_u32 v0, s1, s1, v0
713 ; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
714 ; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
715 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, v3
716 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
717 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
718 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
719 ; GFX11-CU-NEXT: s_endpgm
720 ;
721 ; GFX12-WGP-LABEL: flat_nontemporal_store_1:
722 ; GFX12-WGP: ; %bb.0: ; %entry
723 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
724 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
725 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
726 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s1
727 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0
728 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[1:2]
729 ; GFX12-WGP-NEXT: s_mov_b32 s0, 0x3ff
730 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
731 ; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s0
732 ; GFX12-WGP-NEXT: s_mov_b32 s0, 2
733 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
734 ; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v3, s0, v0
735 ; GFX12-WGP-NEXT: s_mov_b32 s0, 0
736 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
737 ; GFX12-WGP-NEXT: ; implicit-def: $sgpr0
738 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
739 ; GFX12-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
740 ; GFX12-WGP-NEXT: v_mov_b32_e32 v4, v0
741 ; GFX12-WGP-NEXT: s_mov_b32 s1, s2
742 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
743 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v3
744 ; GFX12-WGP-NEXT: s_mov_b32 s0, s3
745 ; GFX12-WGP-NEXT: s_wait_alu 0xfffe
746 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v4
747 ; GFX12-WGP-NEXT: v_add_co_u32 v0, s1, s1, v0
748 ; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
749 ; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
750 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v3
751 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
752 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
753 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
754 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
755 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
756 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
757 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
758 ; GFX12-WGP-NEXT: s_endpgm
759 ;
760 ; GFX12-CU-LABEL: flat_nontemporal_store_1:
761 ; GFX12-CU: ; %bb.0: ; %entry
762 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
763 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
764 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
765 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s1
766 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0
767 ; GFX12-CU-NEXT: flat_load_b32 v2, v[1:2]
768 ; GFX12-CU-NEXT: s_mov_b32 s0, 0x3ff
769 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
770 ; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s0
771 ; GFX12-CU-NEXT: s_mov_b32 s0, 2
772 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
773 ; GFX12-CU-NEXT: v_lshlrev_b32_e64 v3, s0, v0
774 ; GFX12-CU-NEXT: s_mov_b32 s0, 0
775 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
776 ; GFX12-CU-NEXT: ; implicit-def: $sgpr0
777 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
778 ; GFX12-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
779 ; GFX12-CU-NEXT: v_mov_b32_e32 v4, v0
780 ; GFX12-CU-NEXT: s_mov_b32 s1, s2
781 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
782 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, v3
783 ; GFX12-CU-NEXT: s_mov_b32 s0, s3
784 ; GFX12-CU-NEXT: s_wait_alu 0xfffe
785 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, v4
786 ; GFX12-CU-NEXT: v_add_co_u32 v0, s1, s1, v0
787 ; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1
788 ; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
789 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, v3
790 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
791 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
792 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
793 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
794 ; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0
795 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
796 ; GFX12-CU-NEXT: s_wait_storecnt 0x0
797 ; GFX12-CU-NEXT: s_endpgm
798 ptr %in, ptr %out) {
799 entry:
800 %tid = call i32 @llvm.amdgcn.workitem.id.x()
801 %val = load i32, ptr %in, align 4
802 %out.gep = getelementptr inbounds i32, ptr %out, i32 %tid
803 store volatile i32 %val, ptr %out.gep
804 ret void
805 }
807 define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
808 ; GFX7-LABEL: flat_volatile_workgroup_acquire_load:
809 ; GFX7: ; %bb.0: ; %entry
810 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
811 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2
812 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
813 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
814 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
815 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
816 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
817 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
818 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
819 ; GFX7-NEXT: s_waitcnt vmcnt(0)
820 ; GFX7-NEXT: flat_store_dword v[0:1], v2
821 ; GFX7-NEXT: s_endpgm
822 ;
823 ; GFX10-WGP-LABEL: flat_volatile_workgroup_acquire_load:
824 ; GFX10-WGP: ; %bb.0: ; %entry
825 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
826 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
827 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
828 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
829 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
830 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
831 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
832 ; GFX10-WGP-NEXT: buffer_gl0_inv
833 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
834 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5
835 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
836 ; GFX10-WGP-NEXT: s_endpgm
837 ;
838 ; GFX10-CU-LABEL: flat_volatile_workgroup_acquire_load:
839 ; GFX10-CU: ; %bb.0: ; %entry
840 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
841 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
842 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
843 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
844 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
845 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
846 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
847 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
848 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5
849 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
850 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
851 ; GFX10-CU-NEXT: s_endpgm
852 ;
853 ; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_acquire_load:
854 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
855 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
856 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
857 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
858 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
859 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
860 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
861 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
862 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
863 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
864 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
865 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
866 ; SKIP-CACHE-INV-NEXT: s_endpgm
867 ;
868 ; GFX11-WGP-LABEL: flat_volatile_workgroup_acquire_load:
869 ; GFX11-WGP: ; %bb.0: ; %entry
870 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
871 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
872 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
873 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
874 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
875 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
876 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
877 ; GFX11-WGP-NEXT: buffer_gl0_inv
878 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
879 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1
880 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
881 ; GFX11-WGP-NEXT: s_endpgm
882 ;
883 ; GFX11-CU-LABEL: flat_volatile_workgroup_acquire_load:
884 ; GFX11-CU: ; %bb.0: ; %entry
885 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
886 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
887 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
888 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
889 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
890 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
891 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
892 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
893 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1
894 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
895 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
896 ; GFX11-CU-NEXT: s_endpgm
897 ;
898 ; GFX12-WGP-LABEL: flat_volatile_workgroup_acquire_load:
899 ; GFX12-WGP: ; %bb.0: ; %entry
900 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
901 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
902 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
903 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
904 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
905 ; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE
906 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
907 ; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE
908 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0
909 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1
910 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2
911 ; GFX12-WGP-NEXT: s_endpgm
912 ;
913 ; GFX12-CU-LABEL: flat_volatile_workgroup_acquire_load:
914 ; GFX12-CU: ; %bb.0: ; %entry
915 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
916 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
917 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
918 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
919 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
920 ; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]
921 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
922 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0
923 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1
924 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
925 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
926 ; GFX12-CU-NEXT: s_endpgm
927 ptr %in, ptr %out) {
928 entry:
929 %val = load atomic volatile i32, ptr %in syncscope("workgroup") acquire, align 4
930 store i32 %val, ptr %out
931 ret void
932 }
934 define amdgpu_kernel void @flat_volatile_workgroup_release_store(
935 ; GFX7-LABEL: flat_volatile_workgroup_release_store:
936 ; GFX7: ; %bb.0: ; %entry
937 ; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0
938 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2
939 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
940 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
941 ; GFX7-NEXT: v_mov_b32_e32 v1, s7
942 ; GFX7-NEXT: v_mov_b32_e32 v2, s4
943 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
944 ; GFX7-NEXT: flat_store_dword v[0:1], v2
945 ; GFX7-NEXT: s_endpgm
946 ;
947 ; GFX10-WGP-LABEL: flat_volatile_workgroup_release_store:
948 ; GFX10-WGP: ; %bb.0: ; %entry
949 ; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
950 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
951 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
952 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6
953 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7
954 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s4
955 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
956 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
957 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
958 ; GFX10-WGP-NEXT: s_endpgm
959 ;
960 ; GFX10-CU-LABEL: flat_volatile_workgroup_release_store:
961 ; GFX10-CU: ; %bb.0: ; %entry
962 ; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
963 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
964 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
965 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6
966 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7
967 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s4
968 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
969 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
970 ; GFX10-CU-NEXT: s_endpgm
971 ;
972 ; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_release_store:
973 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
974 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0
975 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
976 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
977 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
978 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
979 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
980 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
981 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
982 ; SKIP-CACHE-INV-NEXT: s_endpgm
983 ;
984 ; GFX11-WGP-LABEL: flat_volatile_workgroup_release_store:
985 ; GFX11-WGP: ; %bb.0: ; %entry
986 ; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
987 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
988 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
989 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2
990 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3
991 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
992 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
993 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
994 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
995 ; GFX11-WGP-NEXT: s_endpgm
996 ;
997 ; GFX11-CU-LABEL: flat_volatile_workgroup_release_store:
998 ; GFX11-CU: ; %bb.0: ; %entry
999 ; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1000 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1001 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1002 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2
1003 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3
1004 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
1005 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1006 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
1007 ; GFX11-CU-NEXT: s_endpgm
1008 ;
1009 ; GFX12-WGP-LABEL: flat_volatile_workgroup_release_store:
1010 ; GFX12-WGP: ; %bb.0: ; %entry
1011 ; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
1012 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1013 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1014 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2
1015 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3
1016 ; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s0
1017 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1018 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1019 ; GFX12-WGP-NEXT: s_wait_storecnt 0x0
1020 ; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0
1021 ; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE
1022 ; GFX12-WGP-NEXT: s_endpgm
1023 ;
1024 ; GFX12-CU-LABEL: flat_volatile_workgroup_release_store:
1025 ; GFX12-CU: ; %bb.0: ; %entry
1026 ; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
1027 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
1028 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1029 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2
1030 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3
1031 ; GFX12-CU-NEXT: v_mov_b32_e32 v2, s0
1032 ; GFX12-CU-NEXT: s_wait_dscnt 0x0
1033 ; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2
1034 ; GFX12-CU-NEXT: s_endpgm
1035 i32 %in, ptr %out) {
1036 entry:
1037 store atomic volatile i32 %in, ptr %out syncscope("workgroup") release, align 4
1038 ret void
1039 }
1041 declare i32 @llvm.amdgcn.workitem.id.x()