| blob | c07c6478da0188db92b49fc47e52d2740a09b5aa |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-WGP %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX11-CU %s
9 define amdgpu_kernel void @flat_nontemporal_load_0(
10 ; GFX7-LABEL: flat_nontemporal_load_0:
11 ; GFX7: ; %bb.0: ; %entry
12 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
13 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
14 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
15 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
16 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
17 ; GFX7-NEXT: s_waitcnt vmcnt(0)
18 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
19 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
20 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
21 ; GFX7-NEXT: flat_store_dword v[0:1], v2
22 ; GFX7-NEXT: s_endpgm
23 ;
24 ; GFX10-WGP-LABEL: flat_nontemporal_load_0:
25 ; GFX10-WGP: ; %bb.0: ; %entry
26 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
27 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
28 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
29 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
30 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
31 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
32 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
33 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
34 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
35 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
36 ; GFX10-WGP-NEXT: s_endpgm
37 ;
38 ; GFX10-CU-LABEL: flat_nontemporal_load_0:
39 ; GFX10-CU: ; %bb.0: ; %entry
40 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
41 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
42 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
43 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
44 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
45 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
46 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
47 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
48 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
49 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
50 ; GFX10-CU-NEXT: s_endpgm
51 ;
52 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_0:
53 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
54 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
55 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
56 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
57 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
58 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
59 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
60 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
61 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
62 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
63 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
64 ; SKIP-CACHE-INV-NEXT: s_endpgm
65 ;
66 ; GFX11-WGP-LABEL: flat_nontemporal_load_0:
67 ; GFX11-WGP: ; %bb.0: ; %entry
68 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
69 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
70 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
71 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
72 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
73 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
74 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
75 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
76 ; GFX11-WGP-NEXT: s_endpgm
77 ;
78 ; GFX11-CU-LABEL: flat_nontemporal_load_0:
79 ; GFX11-CU: ; %bb.0: ; %entry
80 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
81 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
82 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
83 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
84 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
85 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
86 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
87 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
88 ; GFX11-CU-NEXT: s_endpgm
89 ptr %in, ptr %out) {
90 entry:
91 %val = load volatile i32, ptr %in, align 4
92 store i32 %val, ptr %out
93 ret void
94 }
96 define amdgpu_kernel void @flat_nontemporal_load_1(
97 ; GFX7-LABEL: flat_nontemporal_load_1:
98 ; GFX7: ; %bb.0: ; %entry
99 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
100 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
101 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
102 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
103 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, s0, v0
104 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
105 ; GFX7-NEXT: flat_load_dword v2, v[0:1] glc
106 ; GFX7-NEXT: s_waitcnt vmcnt(0)
107 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
108 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
109 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
110 ; GFX7-NEXT: flat_store_dword v[0:1], v2
111 ; GFX7-NEXT: s_endpgm
112 ;
113 ; GFX10-WGP-LABEL: flat_nontemporal_load_1:
114 ; GFX10-WGP: ; %bb.0: ; %entry
115 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
116 ; GFX10-WGP-NEXT: v_lshlrev_b32_e32 v0, 2, v0
117 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
118 ; GFX10-WGP-NEXT: v_add_co_u32 v0, s0, s0, v0
119 ; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
120 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc
121 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
122 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
123 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
124 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
125 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
126 ; GFX10-WGP-NEXT: s_endpgm
127 ;
128 ; GFX10-CU-LABEL: flat_nontemporal_load_1:
129 ; GFX10-CU: ; %bb.0: ; %entry
130 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
131 ; GFX10-CU-NEXT: v_lshlrev_b32_e32 v0, 2, v0
132 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
133 ; GFX10-CU-NEXT: v_add_co_u32 v0, s0, s0, v0
134 ; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
135 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc
136 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
137 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
138 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
139 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
140 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
141 ; GFX10-CU-NEXT: s_endpgm
142 ;
143 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:
144 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
145 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
146 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e32 v0, 2, v0
147 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
148 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
149 ; SKIP-CACHE-INV-NEXT: v_add_i32_e32 v0, vcc, s0, v0
150 ; SKIP-CACHE-INV-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
151 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc
152 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
153 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
154 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
155 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
156 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
157 ; SKIP-CACHE-INV-NEXT: s_endpgm
158 ;
159 ; GFX11-WGP-LABEL: flat_nontemporal_load_1:
160 ; GFX11-WGP: ; %bb.0: ; %entry
161 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
162 ; GFX11-WGP-NEXT: v_lshlrev_b32_e32 v0, 2, v0
163 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
164 ; GFX11-WGP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
165 ; GFX11-WGP-NEXT: v_add_co_u32 v0, s0, s0, v0
166 ; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
167 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc
168 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
169 ; GFX11-WGP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
170 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
171 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
172 ; GFX11-WGP-NEXT: s_endpgm
173 ;
174 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
175 ; GFX11-CU: ; %bb.0: ; %entry
176 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
177 ; GFX11-CU-NEXT: v_lshlrev_b32_e32 v0, 2, v0
178 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
179 ; GFX11-CU-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
180 ; GFX11-CU-NEXT: v_add_co_u32 v0, s0, s0, v0
181 ; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
182 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc
183 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
184 ; GFX11-CU-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
185 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
186 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
187 ; GFX11-CU-NEXT: s_endpgm
188 ptr %in, ptr %out) {
189 entry:
190 %tid = call i32 @llvm.amdgcn.workitem.id.x()
191 %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid
192 %val = load volatile i32, ptr %val.gep, align 4
193 store i32 %val, ptr %out
194 ret void
195 }
197 define amdgpu_kernel void @flat_nontemporal_store_0(
198 ; GFX7-LABEL: flat_nontemporal_store_0:
199 ; GFX7: ; %bb.0: ; %entry
200 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
201 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
202 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
203 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
204 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
205 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
206 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
207 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
208 ; GFX7-NEXT: flat_store_dword v[0:1], v2
209 ; GFX7-NEXT: s_waitcnt vmcnt(0)
210 ; GFX7-NEXT: s_endpgm
211 ;
212 ; GFX10-WGP-LABEL: flat_nontemporal_store_0:
213 ; GFX10-WGP: ; %bb.0: ; %entry
214 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
215 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
216 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
217 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
218 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]
219 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
220 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
221 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
222 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
223 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
224 ; GFX10-WGP-NEXT: s_endpgm
225 ;
226 ; GFX10-CU-LABEL: flat_nontemporal_store_0:
227 ; GFX10-CU: ; %bb.0: ; %entry
228 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
229 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
230 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
231 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
232 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
233 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
234 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
235 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
236 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
237 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
238 ; GFX10-CU-NEXT: s_endpgm
239 ;
240 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_0:
241 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
242 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
243 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
244 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
245 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
246 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
247 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
248 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
249 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
250 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
251 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
252 ; SKIP-CACHE-INV-NEXT: s_endpgm
253 ;
254 ; GFX11-WGP-LABEL: flat_nontemporal_store_0:
255 ; GFX11-WGP: ; %bb.0: ; %entry
256 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
257 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
258 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
259 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]
260 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
261 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
262 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc
263 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
264 ; GFX11-WGP-NEXT: s_endpgm
265 ;
266 ; GFX11-CU-LABEL: flat_nontemporal_store_0:
267 ; GFX11-CU: ; %bb.0: ; %entry
268 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
269 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
270 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
271 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
272 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
273 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
274 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
275 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
276 ; GFX11-CU-NEXT: s_endpgm
277 ptr %in, ptr %out) {
278 entry:
279 %val = load i32, ptr %in, align 4
280 store volatile i32 %val, ptr %out
281 ret void
282 }
284 define amdgpu_kernel void @flat_nontemporal_store_1(
285 ; GFX7-LABEL: flat_nontemporal_store_1:
286 ; GFX7: ; %bb.0: ; %entry
287 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
288 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
289 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
290 ; GFX7-NEXT: v_mov_b32_e32 v1, s0
291 ; GFX7-NEXT: v_mov_b32_e32 v2, s1
292 ; GFX7-NEXT: flat_load_dword v2, v[1:2]
293 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
294 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, s2, v0
295 ; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
296 ; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
297 ; GFX7-NEXT: flat_store_dword v[0:1], v2
298 ; GFX7-NEXT: s_waitcnt vmcnt(0)
299 ; GFX7-NEXT: s_endpgm
300 ;
301 ; GFX10-WGP-LABEL: flat_nontemporal_store_1:
302 ; GFX10-WGP: ; %bb.0: ; %entry
303 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
304 ; GFX10-WGP-NEXT: v_lshlrev_b32_e32 v0, 2, v0
305 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
306 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s0
307 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s1
308 ; GFX10-WGP-NEXT: v_add_co_u32 v0, s0, s2, v0
309 ; GFX10-WGP-NEXT: flat_load_dword v2, v[1:2]
310 ; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
311 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
312 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
313 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
314 ; GFX10-WGP-NEXT: s_endpgm
315 ;
316 ; GFX10-CU-LABEL: flat_nontemporal_store_1:
317 ; GFX10-CU: ; %bb.0: ; %entry
318 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
319 ; GFX10-CU-NEXT: v_lshlrev_b32_e32 v0, 2, v0
320 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
321 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s0
322 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s1
323 ; GFX10-CU-NEXT: v_add_co_u32 v0, s0, s2, v0
324 ; GFX10-CU-NEXT: flat_load_dword v2, v[1:2]
325 ; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
326 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
327 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
328 ; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0
329 ; GFX10-CU-NEXT: s_endpgm
330 ;
331 ; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
332 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
333 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
334 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e32 v0, 2, v0
335 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
336 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0
337 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s1
338 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[1:2]
339 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
340 ; SKIP-CACHE-INV-NEXT: v_add_i32_e32 v0, vcc, s2, v0
341 ; SKIP-CACHE-INV-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
342 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
343 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
344 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
345 ; SKIP-CACHE-INV-NEXT: s_endpgm
346 ;
347 ; GFX11-WGP-LABEL: flat_nontemporal_store_1:
348 ; GFX11-WGP: ; %bb.0: ; %entry
349 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
350 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
351 ; GFX11-WGP-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
352 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s1
353 ; GFX11-WGP-NEXT: s_delay_alu instid0(VALU_DEP_2)
354 ; GFX11-WGP-NEXT: v_add_co_u32 v0, s0, s2, v0
355 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[1:2]
356 ; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
357 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
358 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc
359 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
360 ; GFX11-WGP-NEXT: s_endpgm
361 ;
362 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
363 ; GFX11-CU: ; %bb.0: ; %entry
364 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
365 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
366 ; GFX11-CU-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_lshlrev_b32 v0, 2, v0
367 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s1
368 ; GFX11-CU-NEXT: s_delay_alu instid0(VALU_DEP_2)
369 ; GFX11-CU-NEXT: v_add_co_u32 v0, s0, s2, v0
370 ; GFX11-CU-NEXT: flat_load_b32 v2, v[1:2]
371 ; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v1, null, s3, 0, s0
372 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
373 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc
374 ; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0
375 ; GFX11-CU-NEXT: s_endpgm
376 ptr %in, ptr %out) {
377 entry:
378 %tid = call i32 @llvm.amdgcn.workitem.id.x()
379 %val = load i32, ptr %in, align 4
380 %out.gep = getelementptr inbounds i32, ptr %out, i32 %tid
381 store volatile i32 %val, ptr %out.gep
382 ret void
383 }
385 define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
386 ; GFX7-LABEL: flat_volatile_workgroup_acquire_load:
387 ; GFX7: ; %bb.0: ; %entry
388 ; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
389 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
390 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
391 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
392 ; GFX7-NEXT: flat_load_dword v2, v[0:1]
393 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
394 ; GFX7-NEXT: v_mov_b32_e32 v0, s2
395 ; GFX7-NEXT: v_mov_b32_e32 v1, s3
396 ; GFX7-NEXT: s_waitcnt vmcnt(0)
397 ; GFX7-NEXT: flat_store_dword v[0:1], v2
398 ; GFX7-NEXT: s_endpgm
399 ;
400 ; GFX10-WGP-LABEL: flat_volatile_workgroup_acquire_load:
401 ; GFX10-WGP: ; %bb.0: ; %entry
402 ; GFX10-WGP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
403 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
404 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
405 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
406 ; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc
407 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
408 ; GFX10-WGP-NEXT: buffer_gl0_inv
409 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s2
410 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s3
411 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
412 ; GFX10-WGP-NEXT: s_endpgm
413 ;
414 ; GFX10-CU-LABEL: flat_volatile_workgroup_acquire_load:
415 ; GFX10-CU: ; %bb.0: ; %entry
416 ; GFX10-CU-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
417 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
418 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
419 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
420 ; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]
421 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
422 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s2
423 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s3
424 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
425 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
426 ; GFX10-CU-NEXT: s_endpgm
427 ;
428 ; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_acquire_load:
429 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
430 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0
431 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
432 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
433 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1
434 ; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]
435 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
436 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
437 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
438 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
439 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
440 ; SKIP-CACHE-INV-NEXT: s_endpgm
441 ;
442 ; GFX11-WGP-LABEL: flat_volatile_workgroup_acquire_load:
443 ; GFX11-WGP: ; %bb.0: ; %entry
444 ; GFX11-WGP-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
445 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
446 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
447 ; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc
448 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
449 ; GFX11-WGP-NEXT: buffer_gl0_inv
450 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
451 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
452 ; GFX11-WGP-NEXT: s_endpgm
453 ;
454 ; GFX11-CU-LABEL: flat_volatile_workgroup_acquire_load:
455 ; GFX11-CU: ; %bb.0: ; %entry
456 ; GFX11-CU-NEXT: s_load_b128 s[0:3], s[0:1], 0x0
457 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
458 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
459 ; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]
460 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
461 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
462 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
463 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
464 ; GFX11-CU-NEXT: s_endpgm
465 ptr %in, ptr %out) {
466 entry:
467 %val = load atomic volatile i32, ptr %in syncscope("workgroup") acquire, align 4
468 store i32 %val, ptr %out
469 ret void
470 }
472 define amdgpu_kernel void @flat_volatile_workgroup_release_store(
473 ; GFX7-LABEL: flat_volatile_workgroup_release_store:
474 ; GFX7: ; %bb.0: ; %entry
475 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
476 ; GFX7-NEXT: s_load_dword s2, s[4:5], 0x0
477 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
478 ; GFX7-NEXT: v_mov_b32_e32 v0, s0
479 ; GFX7-NEXT: v_mov_b32_e32 v1, s1
480 ; GFX7-NEXT: v_mov_b32_e32 v2, s2
481 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
482 ; GFX7-NEXT: flat_store_dword v[0:1], v2
483 ; GFX7-NEXT: s_endpgm
484 ;
485 ; GFX10-WGP-LABEL: flat_volatile_workgroup_release_store:
486 ; GFX10-WGP: ; %bb.0: ; %entry
487 ; GFX10-WGP-NEXT: s_clause 0x1
488 ; GFX10-WGP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
489 ; GFX10-WGP-NEXT: s_load_dword s2, s[4:5], 0x0
490 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
491 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s0
492 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s1
493 ; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s2
494 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
495 ; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0
496 ; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2
497 ; GFX10-WGP-NEXT: s_endpgm
498 ;
499 ; GFX10-CU-LABEL: flat_volatile_workgroup_release_store:
500 ; GFX10-CU: ; %bb.0: ; %entry
501 ; GFX10-CU-NEXT: s_clause 0x1
502 ; GFX10-CU-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
503 ; GFX10-CU-NEXT: s_load_dword s2, s[4:5], 0x0
504 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
505 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s0
506 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s1
507 ; GFX10-CU-NEXT: v_mov_b32_e32 v2, s2
508 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
509 ; GFX10-CU-NEXT: flat_store_dword v[0:1], v2
510 ; GFX10-CU-NEXT: s_endpgm
511 ;
512 ; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_release_store:
513 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
514 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2
515 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x0
516 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
517 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2
518 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3
519 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s0
520 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
521 ; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2
522 ; SKIP-CACHE-INV-NEXT: s_endpgm
523 ;
524 ; GFX11-WGP-LABEL: flat_volatile_workgroup_release_store:
525 ; GFX11-WGP: ; %bb.0: ; %entry
526 ; GFX11-WGP-NEXT: s_clause 0x1
527 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
528 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
529 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
530 ; GFX11-WGP-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
531 ; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s0
532 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
533 ; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0
534 ; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2
535 ; GFX11-WGP-NEXT: s_endpgm
536 ;
537 ; GFX11-CU-LABEL: flat_volatile_workgroup_release_store:
538 ; GFX11-CU: ; %bb.0: ; %entry
539 ; GFX11-CU-NEXT: s_clause 0x1
540 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x8
541 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
542 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
543 ; GFX11-CU-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
544 ; GFX11-CU-NEXT: v_mov_b32_e32 v2, s0
545 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
546 ; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2
547 ; GFX11-CU-NEXT: s_endpgm
548 i32 %in, ptr %out) {
549 entry:
550 store atomic volatile i32 %in, ptr %out syncscope("workgroup") release, align 4
551 ret void
552 }
554 declare i32 @llvm.amdgcn.workitem.id.x()