| blob | 4e08065e879fd8162e231f0d0e28870bd5632296 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx600 < %s | FileCheck --check-prefixes=GFX6 %s
3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s
4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s
5 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s
6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
9 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 < %s | FileCheck -check-prefixes=GFX940-NOTTGSPLIT %s
10 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx940 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX940-TGSPLIT %s
11 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s
12 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
13 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
14 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
16 define amdgpu_kernel void @private_nontemporal_load_0(
17 ; GFX6-LABEL: private_nontemporal_load_0:
18 ; GFX6: ; %bb.0: ; %entry
19 ; GFX6-NEXT: s_add_u32 s0, s0, s13
20 ; GFX6-NEXT: s_addc_u32 s1, s1, 0
21 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
22 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
23 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
24 ; GFX6-NEXT: s_mov_b32 s11, s5
25 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
26 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
27 ; GFX6-NEXT: s_mov_b32 s10, -1
28 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
29 ; GFX6-NEXT: s_mov_b32 s5, s11
30 ; GFX6-NEXT: s_mov_b32 s6, s10
31 ; GFX6-NEXT: s_mov_b32 s7, s9
32 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
33 ; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen glc slc
34 ; GFX6-NEXT: s_waitcnt vmcnt(0)
35 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
36 ; GFX6-NEXT: s_endpgm
37 ;
38 ; GFX7-LABEL: private_nontemporal_load_0:
39 ; GFX7: ; %bb.0: ; %entry
40 ; GFX7-NEXT: s_add_u32 s0, s0, s13
41 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
42 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
43 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x0
44 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
45 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
46 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
47 ; GFX7-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen glc slc
48 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
49 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
50 ; GFX7-NEXT: s_waitcnt vmcnt(0)
51 ; GFX7-NEXT: flat_store_dword v[0:1], v2
52 ; GFX7-NEXT: s_endpgm
53 ;
54 ; GFX10-WGP-LABEL: private_nontemporal_load_0:
55 ; GFX10-WGP: ; %bb.0: ; %entry
56 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, s13
57 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
58 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
59 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
60 ; GFX10-WGP-NEXT: s_nop 0
61 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
62 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
63 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
64 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
65 ; GFX10-WGP-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
66 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
67 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
68 ; GFX10-WGP-NEXT: s_endpgm
69 ;
70 ; GFX10-CU-LABEL: private_nontemporal_load_0:
71 ; GFX10-CU: ; %bb.0: ; %entry
72 ; GFX10-CU-NEXT: s_add_u32 s0, s0, s13
73 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
74 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
75 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
76 ; GFX10-CU-NEXT: s_nop 0
77 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
78 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
79 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
80 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
81 ; GFX10-CU-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
82 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
83 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
84 ; GFX10-CU-NEXT: s_endpgm
85 ;
86 ; SKIP-CACHE-INV-LABEL: private_nontemporal_load_0:
87 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
88 ; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[12:13]
89 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s12, s0
90 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x0
91 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
92 ; SKIP-CACHE-INV-NEXT: s_add_u32 s12, s12, s9
93 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s13, s13, 0
94 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
95 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
96 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
97 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
98 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
99 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
100 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
101 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
102 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
103 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
104 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
105 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
106 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, v0, s[12:15], 0 offen glc slc
107 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
108 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
109 ; SKIP-CACHE-INV-NEXT: s_endpgm
110 ;
111 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_load_0:
112 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
113 ; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s0, s0, s13
114 ; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s1, s1, 0
115 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
116 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
117 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
118 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
119 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
120 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
121 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
122 ; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
123 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
124 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
125 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
126 ;
127 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_load_0:
128 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
129 ; GFX90A-TGSPLIT-NEXT: s_add_u32 s0, s0, s13
130 ; GFX90A-TGSPLIT-NEXT: s_addc_u32 s1, s1, 0
131 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
132 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
133 ; GFX90A-TGSPLIT-NEXT: s_nop 0
134 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
135 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
136 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
137 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
138 ; GFX90A-TGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
139 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
140 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
141 ; GFX90A-TGSPLIT-NEXT: s_endpgm
142 ;
143 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_load_0:
144 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
145 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
146 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
147 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
148 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
149 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
150 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
151 ; GFX940-NOTTGSPLIT-NEXT: scratch_load_dword v1, off, s2 nt
152 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
153 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
154 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
155 ;
156 ; GFX940-TGSPLIT-LABEL: private_nontemporal_load_0:
157 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
158 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
159 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
160 ; GFX940-TGSPLIT-NEXT: s_nop 0
161 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
162 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
163 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
164 ; GFX940-TGSPLIT-NEXT: scratch_load_dword v1, off, s2 nt
165 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
166 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
167 ; GFX940-TGSPLIT-NEXT: s_endpgm
168 ;
169 ; GFX11-WGP-LABEL: private_nontemporal_load_0:
170 ; GFX11-WGP: ; %bb.0: ; %entry
171 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
172 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
173 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
174 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
175 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
176 ; GFX11-WGP-NEXT: scratch_load_b32 v1, off, s2 slc dlc
177 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
178 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
179 ; GFX11-WGP-NEXT: s_endpgm
180 ;
181 ; GFX11-CU-LABEL: private_nontemporal_load_0:
182 ; GFX11-CU: ; %bb.0: ; %entry
183 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
184 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
185 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
186 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
187 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
188 ; GFX11-CU-NEXT: scratch_load_b32 v1, off, s2 slc dlc
189 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
190 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
191 ; GFX11-CU-NEXT: s_endpgm
192 ;
193 ; GFX12-WGP-LABEL: private_nontemporal_load_0:
194 ; GFX12-WGP: ; %bb.0: ; %entry
195 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
196 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
197 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
198 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
199 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
200 ; GFX12-WGP-NEXT: scratch_load_b32 v1, off, s2 th:TH_LOAD_NT
201 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
202 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
203 ; GFX12-WGP-NEXT: s_endpgm
204 ;
205 ; GFX12-CU-LABEL: private_nontemporal_load_0:
206 ; GFX12-CU: ; %bb.0: ; %entry
207 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
208 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
209 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
210 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
211 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
212 ; GFX12-CU-NEXT: scratch_load_b32 v1, off, s2 th:TH_LOAD_NT
213 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
214 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
215 ; GFX12-CU-NEXT: s_endpgm
216 ptr addrspace(5) %in, ptr addrspace(1) %out) {
217 entry:
218 %val = load i32, ptr addrspace(5) %in, align 4, !nontemporal !0
219 store i32 %val, ptr addrspace(1) %out
220 ret void
221 }
223 define amdgpu_kernel void @private_nontemporal_load_1(
224 ; GFX6-LABEL: private_nontemporal_load_1:
225 ; GFX6: ; %bb.0: ; %entry
226 ; GFX6-NEXT: s_add_u32 s0, s0, s13
227 ; GFX6-NEXT: s_addc_u32 s1, s1, 0
228 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
229 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
230 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
231 ; GFX6-NEXT: s_mov_b32 s11, s5
232 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
233 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
234 ; GFX6-NEXT: s_mov_b32 s10, -1
235 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
236 ; GFX6-NEXT: s_mov_b32 s5, s11
237 ; GFX6-NEXT: s_mov_b32 s6, s10
238 ; GFX6-NEXT: s_mov_b32 s7, s9
239 ; GFX6-NEXT: s_mov_b32 s9, 2
240 ; GFX6-NEXT: v_lshlrev_b32_e64 v0, s9, v0
241 ; GFX6-NEXT: v_add_i32_e64 v0, s[8:9], s8, v0
242 ; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen glc slc
243 ; GFX6-NEXT: s_waitcnt vmcnt(0)
244 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
245 ; GFX6-NEXT: s_endpgm
246 ;
247 ; GFX7-LABEL: private_nontemporal_load_1:
248 ; GFX7: ; %bb.0: ; %entry
249 ; GFX7-NEXT: s_add_u32 s0, s0, s13
250 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
251 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
252 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x0
253 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
254 ; GFX7-NEXT: s_mov_b32 s7, 2
255 ; GFX7-NEXT: v_lshlrev_b32_e64 v0, s7, v0
256 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
257 ; GFX7-NEXT: v_add_i32_e64 v0, s[6:7], s6, v0
258 ; GFX7-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen glc slc
259 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
260 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
261 ; GFX7-NEXT: s_waitcnt vmcnt(0)
262 ; GFX7-NEXT: flat_store_dword v[0:1], v2
263 ; GFX7-NEXT: s_endpgm
264 ;
265 ; GFX10-WGP-LABEL: private_nontemporal_load_1:
266 ; GFX10-WGP: ; %bb.0: ; %entry
267 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, s13
268 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
269 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
270 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v0
271 ; GFX10-WGP-NEXT: s_load_dword s7, s[4:5], 0x0
272 ; GFX10-WGP-NEXT: s_nop 0
273 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
274 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
275 ; GFX10-WGP-NEXT: s_mov_b32 s6, 2
276 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
277 ; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, s6, s7
278 ; GFX10-WGP-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
279 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
280 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
281 ; GFX10-WGP-NEXT: s_endpgm
282 ;
283 ; GFX10-CU-LABEL: private_nontemporal_load_1:
284 ; GFX10-CU: ; %bb.0: ; %entry
285 ; GFX10-CU-NEXT: s_add_u32 s0, s0, s13
286 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
287 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
288 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, v0
289 ; GFX10-CU-NEXT: s_load_dword s7, s[4:5], 0x0
290 ; GFX10-CU-NEXT: s_nop 0
291 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
292 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
293 ; GFX10-CU-NEXT: s_mov_b32 s6, 2
294 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
295 ; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, s6, s7
296 ; GFX10-CU-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
297 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
298 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
299 ; GFX10-CU-NEXT: s_endpgm
300 ;
301 ; SKIP-CACHE-INV-LABEL: private_nontemporal_load_1:
302 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
303 ; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[12:13]
304 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s12, s0
305 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x0
306 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
307 ; SKIP-CACHE-INV-NEXT: s_add_u32 s12, s12, s9
308 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s13, s13, 0
309 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
310 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
311 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
312 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
313 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
314 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
315 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
316 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
317 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
318 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
319 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
320 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 2
321 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v0, s5, v0
322 ; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v0, s[4:5], s4, v0
323 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, v0, s[12:15], 0 offen glc slc
324 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
325 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
326 ; SKIP-CACHE-INV-NEXT: s_endpgm
327 ;
328 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_load_1:
329 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
330 ; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s0, s0, s13
331 ; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s1, s1, 0
332 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
333 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v0
334 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[4:5], 0x0
335 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
336 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
337 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
338 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
339 ; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
340 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 2
341 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
342 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
343 ; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
344 ; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
345 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
346 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
347 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
348 ;
349 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_load_1:
350 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
351 ; GFX90A-TGSPLIT-NEXT: s_add_u32 s0, s0, s13
352 ; GFX90A-TGSPLIT-NEXT: s_addc_u32 s1, s1, 0
353 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
354 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, v0
355 ; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[4:5], 0x0
356 ; GFX90A-TGSPLIT-NEXT: s_nop 0
357 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
358 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
359 ; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
360 ; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
361 ; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 2
362 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
363 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
364 ; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
365 ; GFX90A-TGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
366 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
367 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
368 ; GFX90A-TGSPLIT-NEXT: s_endpgm
369 ;
370 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_load_1:
371 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
372 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
373 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, v0
374 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s3, s[0:1], 0x0
375 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
376 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
377 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
378 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
379 ; GFX940-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
380 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b32 s2, 2
381 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
382 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
383 ; GFX940-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
384 ; GFX940-NOTTGSPLIT-NEXT: scratch_load_dword v1, v1, off nt
385 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
386 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
387 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
388 ;
389 ; GFX940-TGSPLIT-LABEL: private_nontemporal_load_1:
390 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
391 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
392 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, v0
393 ; GFX940-TGSPLIT-NEXT: s_load_dword s3, s[0:1], 0x0
394 ; GFX940-TGSPLIT-NEXT: s_nop 0
395 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
396 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
397 ; GFX940-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
398 ; GFX940-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
399 ; GFX940-TGSPLIT-NEXT: s_mov_b32 s2, 2
400 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
401 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
402 ; GFX940-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
403 ; GFX940-TGSPLIT-NEXT: scratch_load_dword v1, v1, off nt
404 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
405 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
406 ; GFX940-TGSPLIT-NEXT: s_endpgm
407 ;
408 ; GFX11-WGP-LABEL: private_nontemporal_load_1:
409 ; GFX11-WGP: ; %bb.0: ; %entry
410 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
411 ; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v0
412 ; GFX11-WGP-NEXT: s_load_b32 s3, s[0:1], 0x0
413 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
414 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
415 ; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
416 ; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
417 ; GFX11-WGP-NEXT: s_mov_b32 s2, 2
418 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
419 ; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
420 ; GFX11-WGP-NEXT: scratch_load_b32 v1, v1, off slc dlc
421 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
422 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
423 ; GFX11-WGP-NEXT: s_endpgm
424 ;
425 ; GFX11-CU-LABEL: private_nontemporal_load_1:
426 ; GFX11-CU: ; %bb.0: ; %entry
427 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
428 ; GFX11-CU-NEXT: v_mov_b32_e32 v1, v0
429 ; GFX11-CU-NEXT: s_load_b32 s3, s[0:1], 0x0
430 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
431 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
432 ; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
433 ; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
434 ; GFX11-CU-NEXT: s_mov_b32 s2, 2
435 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
436 ; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
437 ; GFX11-CU-NEXT: scratch_load_b32 v1, v1, off slc dlc
438 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
439 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
440 ; GFX11-CU-NEXT: s_endpgm
441 ;
442 ; GFX12-WGP-LABEL: private_nontemporal_load_1:
443 ; GFX12-WGP: ; %bb.0: ; %entry
444 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
445 ; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v0
446 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
447 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
448 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
449 ; GFX12-WGP-NEXT: s_mov_b32 s3, 0x3ff
450 ; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s3
451 ; GFX12-WGP-NEXT: s_mov_b32 s3, 2
452 ; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v1, s3, v1
453 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
454 ; GFX12-WGP-NEXT: scratch_load_b32 v1, v1, s2 th:TH_LOAD_NT
455 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
456 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
457 ; GFX12-WGP-NEXT: s_endpgm
458 ;
459 ; GFX12-CU-LABEL: private_nontemporal_load_1:
460 ; GFX12-CU: ; %bb.0: ; %entry
461 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
462 ; GFX12-CU-NEXT: v_mov_b32_e32 v1, v0
463 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
464 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
465 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
466 ; GFX12-CU-NEXT: s_mov_b32 s3, 0x3ff
467 ; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s3
468 ; GFX12-CU-NEXT: s_mov_b32 s3, 2
469 ; GFX12-CU-NEXT: v_lshlrev_b32_e64 v1, s3, v1
470 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
471 ; GFX12-CU-NEXT: scratch_load_b32 v1, v1, s2 th:TH_LOAD_NT
472 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
473 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
474 ; GFX12-CU-NEXT: s_endpgm
475 ptr addrspace(5) %in, ptr addrspace(1) %out) {
476 entry:
477 %tid = call i32 @llvm.amdgcn.workitem.id.x()
478 %val.gep = getelementptr inbounds i32, ptr addrspace(5) %in, i32 %tid
479 %val = load i32, ptr addrspace(5) %val.gep, align 4, !nontemporal !0
480 store i32 %val, ptr addrspace(1) %out
481 ret void
482 }
484 define amdgpu_kernel void @private_nontemporal_store_0(
485 ; GFX6-LABEL: private_nontemporal_store_0:
486 ; GFX6: ; %bb.0: ; %entry
487 ; GFX6-NEXT: s_add_u32 s0, s0, s13
488 ; GFX6-NEXT: s_addc_u32 s1, s1, 0
489 ; GFX6-NEXT: s_mov_b64 s[4:5], s[6:7]
490 ; GFX6-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
491 ; GFX6-NEXT: s_load_dword s4, s[4:5], 0x2
492 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
493 ; GFX6-NEXT: s_load_dword s5, s[6:7], 0x0
494 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
495 ; GFX6-NEXT: v_mov_b32_e32 v0, s5
496 ; GFX6-NEXT: v_mov_b32_e32 v1, s4
497 ; GFX6-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
498 ; GFX6-NEXT: s_endpgm
499 ;
500 ; GFX7-LABEL: private_nontemporal_store_0:
501 ; GFX7: ; %bb.0: ; %entry
502 ; GFX7-NEXT: s_add_u32 s0, s0, s13
503 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
504 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
505 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
506 ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x2
507 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
508 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0
509 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
510 ; GFX7-NEXT: v_mov_b32_e32 v0, s5
511 ; GFX7-NEXT: v_mov_b32_e32 v1, s4
512 ; GFX7-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
513 ; GFX7-NEXT: s_endpgm
514 ;
515 ; GFX10-WGP-LABEL: private_nontemporal_store_0:
516 ; GFX10-WGP: ; %bb.0: ; %entry
517 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, s13
518 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
519 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
520 ; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
521 ; GFX10-WGP-NEXT: s_nop 0
522 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x8
523 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
524 ; GFX10-WGP-NEXT: s_load_dword s5, s[6:7], 0x0
525 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
526 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5
527 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4
528 ; GFX10-WGP-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
529 ; GFX10-WGP-NEXT: s_endpgm
530 ;
531 ; GFX10-CU-LABEL: private_nontemporal_store_0:
532 ; GFX10-CU: ; %bb.0: ; %entry
533 ; GFX10-CU-NEXT: s_add_u32 s0, s0, s13
534 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
535 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
536 ; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
537 ; GFX10-CU-NEXT: s_nop 0
538 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x8
539 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
540 ; GFX10-CU-NEXT: s_load_dword s5, s[6:7], 0x0
541 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
542 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5
543 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4
544 ; GFX10-CU-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
545 ; GFX10-CU-NEXT: s_endpgm
546 ;
547 ; SKIP-CACHE-INV-LABEL: private_nontemporal_store_0:
548 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
549 ; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[12:13]
550 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s12, s0
551 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x0
552 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
553 ; SKIP-CACHE-INV-NEXT: s_add_u32 s12, s12, s9
554 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s13, s13, 0
555 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
556 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
557 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[0:1], 0x2
558 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
559 ; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[2:3], 0x0
560 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
561 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1
562 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0
563 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen glc slc
564 ; SKIP-CACHE-INV-NEXT: s_endpgm
565 ;
566 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_store_0:
567 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
568 ; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s0, s0, s13
569 ; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s1, s1, 0
570 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
571 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
572 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
573 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
574 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
575 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[6:7], 0x0
576 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
577 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s5
578 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s4
579 ; GFX90A-NOTTGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
580 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
581 ;
582 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_store_0:
583 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
584 ; GFX90A-TGSPLIT-NEXT: s_add_u32 s0, s0, s13
585 ; GFX90A-TGSPLIT-NEXT: s_addc_u32 s1, s1, 0
586 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
587 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
588 ; GFX90A-TGSPLIT-NEXT: s_nop 0
589 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x8
590 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
591 ; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[6:7], 0x0
592 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
593 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5
594 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s4
595 ; GFX90A-TGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
596 ; GFX90A-TGSPLIT-NEXT: s_endpgm
597 ;
598 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_store_0:
599 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
600 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
601 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
602 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
603 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
604 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
605 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s1, s[2:3], 0x0
606 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
607 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1
608 ; GFX940-NOTTGSPLIT-NEXT: scratch_store_dword off, v0, s0 sc0 nt sc1
609 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
610 ;
611 ; GFX940-TGSPLIT-LABEL: private_nontemporal_store_0:
612 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
613 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
614 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
615 ; GFX940-TGSPLIT-NEXT: s_nop 0
616 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x8
617 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
618 ; GFX940-TGSPLIT-NEXT: s_load_dword s1, s[2:3], 0x0
619 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
620 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1
621 ; GFX940-TGSPLIT-NEXT: scratch_store_dword off, v0, s0 sc0 nt sc1
622 ; GFX940-TGSPLIT-NEXT: s_endpgm
623 ;
624 ; GFX11-WGP-LABEL: private_nontemporal_store_0:
625 ; GFX11-WGP: ; %bb.0: ; %entry
626 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
627 ; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
628 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
629 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
630 ; GFX11-WGP-NEXT: s_load_b32 s1, s[2:3], 0x0
631 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
632 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1
633 ; GFX11-WGP-NEXT: scratch_store_b32 off, v0, s0 glc slc dlc
634 ; GFX11-WGP-NEXT: s_endpgm
635 ;
636 ; GFX11-CU-LABEL: private_nontemporal_store_0:
637 ; GFX11-CU: ; %bb.0: ; %entry
638 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
639 ; GFX11-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
640 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
641 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
642 ; GFX11-CU-NEXT: s_load_b32 s1, s[2:3], 0x0
643 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
644 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1
645 ; GFX11-CU-NEXT: scratch_store_b32 off, v0, s0 glc slc dlc
646 ; GFX11-CU-NEXT: s_endpgm
647 ;
648 ; GFX12-WGP-LABEL: private_nontemporal_store_0:
649 ; GFX12-WGP: ; %bb.0: ; %entry
650 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
651 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
652 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
653 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
654 ; GFX12-WGP-NEXT: s_load_b32 s1, s[2:3], 0x0
655 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
656 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1
657 ; GFX12-WGP-NEXT: scratch_store_b32 off, v0, s0 th:TH_STORE_NT
658 ; GFX12-WGP-NEXT: s_endpgm
659 ;
660 ; GFX12-CU-LABEL: private_nontemporal_store_0:
661 ; GFX12-CU: ; %bb.0: ; %entry
662 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
663 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
664 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
665 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
666 ; GFX12-CU-NEXT: s_load_b32 s1, s[2:3], 0x0
667 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
668 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1
669 ; GFX12-CU-NEXT: scratch_store_b32 off, v0, s0 th:TH_STORE_NT
670 ; GFX12-CU-NEXT: s_endpgm
671 ptr addrspace(1) %in, ptr addrspace(5) %out) {
672 entry:
673 %val = load i32, ptr addrspace(1) %in, align 4
674 store i32 %val, ptr addrspace(5) %out, !nontemporal !0
675 ret void
676 }
678 define amdgpu_kernel void @private_nontemporal_store_1(
679 ; GFX6-LABEL: private_nontemporal_store_1:
680 ; GFX6: ; %bb.0: ; %entry
681 ; GFX6-NEXT: s_add_u32 s0, s0, s13
682 ; GFX6-NEXT: s_addc_u32 s1, s1, 0
683 ; GFX6-NEXT: s_mov_b64 s[4:5], s[6:7]
684 ; GFX6-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
685 ; GFX6-NEXT: s_load_dword s5, s[4:5], 0x2
686 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
687 ; GFX6-NEXT: s_load_dword s4, s[6:7], 0x0
688 ; GFX6-NEXT: s_mov_b32 s6, 2
689 ; GFX6-NEXT: v_lshlrev_b32_e64 v0, s6, v0
690 ; GFX6-NEXT: v_add_i32_e64 v1, s[6:7], s5, v0
691 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
692 ; GFX6-NEXT: v_mov_b32_e32 v0, s4
693 ; GFX6-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
694 ; GFX6-NEXT: s_endpgm
695 ;
696 ; GFX7-LABEL: private_nontemporal_store_1:
697 ; GFX7: ; %bb.0: ; %entry
698 ; GFX7-NEXT: s_add_u32 s0, s0, s13
699 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
700 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
701 ; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
702 ; GFX7-NEXT: s_load_dword s5, s[4:5], 0x2
703 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
704 ; GFX7-NEXT: s_load_dword s4, s[6:7], 0x0
705 ; GFX7-NEXT: s_mov_b32 s6, 2
706 ; GFX7-NEXT: v_lshlrev_b32_e64 v0, s6, v0
707 ; GFX7-NEXT: v_add_i32_e64 v1, s[6:7], s5, v0
708 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
709 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
710 ; GFX7-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
711 ; GFX7-NEXT: s_endpgm
712 ;
713 ; GFX10-WGP-LABEL: private_nontemporal_store_1:
714 ; GFX10-WGP: ; %bb.0: ; %entry
715 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, s13
716 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
717 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
718 ; GFX10-WGP-NEXT: s_nop 0
719 ; GFX10-WGP-NEXT: s_load_dword s6, s[6:7], 0x8
720 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
721 ; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
722 ; GFX10-WGP-NEXT: s_mov_b32 s5, 2
723 ; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v0, s5, s6
724 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
725 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
726 ; GFX10-WGP-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
727 ; GFX10-WGP-NEXT: s_endpgm
728 ;
729 ; GFX10-CU-LABEL: private_nontemporal_store_1:
730 ; GFX10-CU: ; %bb.0: ; %entry
731 ; GFX10-CU-NEXT: s_add_u32 s0, s0, s13
732 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
733 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
734 ; GFX10-CU-NEXT: s_nop 0
735 ; GFX10-CU-NEXT: s_load_dword s6, s[6:7], 0x8
736 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
737 ; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
738 ; GFX10-CU-NEXT: s_mov_b32 s5, 2
739 ; GFX10-CU-NEXT: v_lshl_add_u32 v1, v0, s5, s6
740 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
741 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
742 ; GFX10-CU-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
743 ; GFX10-CU-NEXT: s_endpgm
744 ;
745 ; SKIP-CACHE-INV-LABEL: private_nontemporal_store_1:
746 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
747 ; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[12:13]
748 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s12, s0
749 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x0
750 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
751 ; SKIP-CACHE-INV-NEXT: s_add_u32 s12, s12, s9
752 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s13, s13, 0
753 ; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[2:3]
754 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
755 ; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[0:1], 0x2
756 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
757 ; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[2:3], 0x0
758 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, 2
759 ; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v0, s2, v0
760 ; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v1, s[2:3], s1, v0
761 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
762 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0
763 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, v1, s[12:15], 0 offen glc slc
764 ; SKIP-CACHE-INV-NEXT: s_endpgm
765 ;
766 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_store_1:
767 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
768 ; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s0, s0, s13
769 ; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s1, s1, 0
770 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
771 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
772 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
773 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
774 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
775 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
776 ; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
777 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 2
778 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
779 ; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s5, v1
780 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
781 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4
782 ; GFX90A-NOTTGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
783 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
784 ;
785 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_store_1:
786 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
787 ; GFX90A-TGSPLIT-NEXT: s_add_u32 s0, s0, s13
788 ; GFX90A-TGSPLIT-NEXT: s_addc_u32 s1, s1, 0
789 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x0
790 ; GFX90A-TGSPLIT-NEXT: s_nop 0
791 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[6:7], 0x8
792 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
793 ; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
794 ; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
795 ; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
796 ; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 2
797 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
798 ; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s5, v1
799 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
800 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4
801 ; GFX90A-TGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
802 ; GFX90A-TGSPLIT-NEXT: s_endpgm
803 ;
804 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_store_1:
805 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
806 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
807 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
808 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
809 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
810 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
811 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
812 ; GFX940-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
813 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b32 s1, 2
814 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
815 ; GFX940-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s1, v1
816 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
817 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
818 ; GFX940-NOTTGSPLIT-NEXT: scratch_store_dword v1, v0, off sc0 nt sc1
819 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
820 ;
821 ; GFX940-TGSPLIT-LABEL: private_nontemporal_store_1:
822 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
823 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
824 ; GFX940-TGSPLIT-NEXT: s_nop 0
825 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[2:3], 0x8
826 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
827 ; GFX940-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
828 ; GFX940-TGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
829 ; GFX940-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
830 ; GFX940-TGSPLIT-NEXT: s_mov_b32 s1, 2
831 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
832 ; GFX940-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s1, v1
833 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
834 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
835 ; GFX940-TGSPLIT-NEXT: scratch_store_dword v1, v0, off sc0 nt sc1
836 ; GFX940-TGSPLIT-NEXT: s_endpgm
837 ;
838 ; GFX11-WGP-LABEL: private_nontemporal_store_1:
839 ; GFX11-WGP: ; %bb.0: ; %entry
840 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
841 ; GFX11-WGP-NEXT: s_load_b32 s2, s[2:3], 0x8
842 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
843 ; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
844 ; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
845 ; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
846 ; GFX11-WGP-NEXT: s_mov_b32 s1, 2
847 ; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v0, s1, s2
848 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
849 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
850 ; GFX11-WGP-NEXT: scratch_store_b32 v1, v0, off glc slc dlc
851 ; GFX11-WGP-NEXT: s_endpgm
852 ;
853 ; GFX11-CU-LABEL: private_nontemporal_store_1:
854 ; GFX11-CU: ; %bb.0: ; %entry
855 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
856 ; GFX11-CU-NEXT: s_load_b32 s2, s[2:3], 0x8
857 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
858 ; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
859 ; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
860 ; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
861 ; GFX11-CU-NEXT: s_mov_b32 s1, 2
862 ; GFX11-CU-NEXT: v_lshl_add_u32 v1, v0, s1, s2
863 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
864 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
865 ; GFX11-CU-NEXT: scratch_store_b32 v1, v0, off glc slc dlc
866 ; GFX11-CU-NEXT: s_endpgm
867 ;
868 ; GFX12-WGP-LABEL: private_nontemporal_store_1:
869 ; GFX12-WGP: ; %bb.0: ; %entry
870 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
871 ; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
872 ; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x8
873 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
874 ; GFX12-WGP-NEXT: s_load_b32 s1, s[2:3], 0x0
875 ; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
876 ; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s2
877 ; GFX12-WGP-NEXT: s_mov_b32 s2, 2
878 ; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v1, s2, v0
879 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
880 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1
881 ; GFX12-WGP-NEXT: scratch_store_b32 v1, v0, s0 th:TH_STORE_NT
882 ; GFX12-WGP-NEXT: s_endpgm
883 ;
884 ; GFX12-CU-LABEL: private_nontemporal_store_1:
885 ; GFX12-CU: ; %bb.0: ; %entry
886 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
887 ; GFX12-CU-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
888 ; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x8
889 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
890 ; GFX12-CU-NEXT: s_load_b32 s1, s[2:3], 0x0
891 ; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
892 ; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s2
893 ; GFX12-CU-NEXT: s_mov_b32 s2, 2
894 ; GFX12-CU-NEXT: v_lshlrev_b32_e64 v1, s2, v0
895 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
896 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1
897 ; GFX12-CU-NEXT: scratch_store_b32 v1, v0, s0 th:TH_STORE_NT
898 ; GFX12-CU-NEXT: s_endpgm
899 ptr addrspace(1) %in, ptr addrspace(5) %out) {
900 entry:
901 %tid = call i32 @llvm.amdgcn.workitem.id.x()
902 %val = load i32, ptr addrspace(1) %in, align 4
903 %out.gep = getelementptr inbounds i32, ptr addrspace(5) %out, i32 %tid
904 store i32 %val, ptr addrspace(5) %out.gep, !nontemporal !0
905 ret void
906 }
908 define amdgpu_kernel void @private_nontemporal_volatile_load(
909 ; GFX6-LABEL: private_nontemporal_volatile_load:
910 ; GFX6: ; %bb.0: ; %entry
911 ; GFX6-NEXT: s_add_u32 s0, s0, s13
912 ; GFX6-NEXT: s_addc_u32 s1, s1, 0
913 ; GFX6-NEXT: s_load_dword s8, s[6:7], 0x0
914 ; GFX6-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x2
915 ; GFX6-NEXT: s_waitcnt lgkmcnt(0)
916 ; GFX6-NEXT: s_mov_b32 s11, s5
917 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
918 ; GFX6-NEXT: s_mov_b32 s9, 0x100f000
919 ; GFX6-NEXT: s_mov_b32 s10, -1
920 ; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5_sgpr6_sgpr7
921 ; GFX6-NEXT: s_mov_b32 s5, s11
922 ; GFX6-NEXT: s_mov_b32 s6, s10
923 ; GFX6-NEXT: s_mov_b32 s7, s9
924 ; GFX6-NEXT: v_mov_b32_e32 v0, s8
925 ; GFX6-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen glc
926 ; GFX6-NEXT: s_waitcnt vmcnt(0)
927 ; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
928 ; GFX6-NEXT: s_endpgm
929 ;
930 ; GFX7-LABEL: private_nontemporal_volatile_load:
931 ; GFX7: ; %bb.0: ; %entry
932 ; GFX7-NEXT: s_add_u32 s0, s0, s13
933 ; GFX7-NEXT: s_addc_u32 s1, s1, 0
934 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
935 ; GFX7-NEXT: s_load_dword s6, s[4:5], 0x0
936 ; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2
937 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
938 ; GFX7-NEXT: v_mov_b32_e32 v0, s6
939 ; GFX7-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen glc
940 ; GFX7-NEXT: s_waitcnt vmcnt(0)
941 ; GFX7-NEXT: v_mov_b32_e32 v0, s4
942 ; GFX7-NEXT: v_mov_b32_e32 v1, s5
943 ; GFX7-NEXT: flat_store_dword v[0:1], v2
944 ; GFX7-NEXT: s_endpgm
945 ;
946 ; GFX10-WGP-LABEL: private_nontemporal_volatile_load:
947 ; GFX10-WGP: ; %bb.0: ; %entry
948 ; GFX10-WGP-NEXT: s_add_u32 s0, s0, s13
949 ; GFX10-WGP-NEXT: s_addc_u32 s1, s1, 0
950 ; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[6:7]
951 ; GFX10-WGP-NEXT: s_load_dword s6, s[4:5], 0x0
952 ; GFX10-WGP-NEXT: s_nop 0
953 ; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
954 ; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
955 ; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
956 ; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s6
957 ; GFX10-WGP-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc dlc
958 ; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)
959 ; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
960 ; GFX10-WGP-NEXT: s_endpgm
961 ;
962 ; GFX10-CU-LABEL: private_nontemporal_volatile_load:
963 ; GFX10-CU: ; %bb.0: ; %entry
964 ; GFX10-CU-NEXT: s_add_u32 s0, s0, s13
965 ; GFX10-CU-NEXT: s_addc_u32 s1, s1, 0
966 ; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[6:7]
967 ; GFX10-CU-NEXT: s_load_dword s6, s[4:5], 0x0
968 ; GFX10-CU-NEXT: s_nop 0
969 ; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
970 ; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
971 ; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
972 ; GFX10-CU-NEXT: v_mov_b32_e32 v1, s6
973 ; GFX10-CU-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc dlc
974 ; GFX10-CU-NEXT: s_waitcnt vmcnt(0)
975 ; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
976 ; GFX10-CU-NEXT: s_endpgm
977 ;
978 ; SKIP-CACHE-INV-LABEL: private_nontemporal_volatile_load:
979 ; SKIP-CACHE-INV: ; %bb.0: ; %entry
980 ; SKIP-CACHE-INV-NEXT: s_getpc_b64 s[12:13]
981 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s12, s0
982 ; SKIP-CACHE-INV-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x0
983 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
984 ; SKIP-CACHE-INV-NEXT: s_add_u32 s12, s12, s9
985 ; SKIP-CACHE-INV-NEXT: s_addc_u32 s13, s13, 0
986 ; SKIP-CACHE-INV-NEXT: s_load_dword s4, s[2:3], 0x0
987 ; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x2
988 ; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)
989 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s7, s1
990 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 killed $sgpr0_sgpr1
991 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s5, 0xf000
992 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s6, -1
993 ; SKIP-CACHE-INV-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1_sgpr2_sgpr3
994 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s1, s7
995 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s6
996 ; SKIP-CACHE-INV-NEXT: s_mov_b32 s3, s5
997 ; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s4
998 ; SKIP-CACHE-INV-NEXT: buffer_load_dword v0, v0, s[12:15], 0 offen glc
999 ; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)
1000 ; SKIP-CACHE-INV-NEXT: buffer_store_dword v0, off, s[0:3], 0
1001 ; SKIP-CACHE-INV-NEXT: s_endpgm
1002 ;
1003 ; GFX90A-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
1004 ; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry
1005 ; GFX90A-NOTTGSPLIT-NEXT: s_add_u32 s0, s0, s13
1006 ; GFX90A-NOTTGSPLIT-NEXT: s_addc_u32 s1, s1, 0
1007 ; GFX90A-NOTTGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1008 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1009 ; GFX90A-NOTTGSPLIT-NEXT: s_nop 0
1010 ; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1011 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1012 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1013 ; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1014 ; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc
1015 ; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
1016 ; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1017 ; GFX90A-NOTTGSPLIT-NEXT: s_endpgm
1018 ;
1019 ; GFX90A-TGSPLIT-LABEL: private_nontemporal_volatile_load:
1020 ; GFX90A-TGSPLIT: ; %bb.0: ; %entry
1021 ; GFX90A-TGSPLIT-NEXT: s_add_u32 s0, s0, s13
1022 ; GFX90A-TGSPLIT-NEXT: s_addc_u32 s1, s1, 0
1023 ; GFX90A-TGSPLIT-NEXT: s_mov_b64 s[4:5], s[6:7]
1024 ; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[4:5], 0x0
1025 ; GFX90A-TGSPLIT-NEXT: s_nop 0
1026 ; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8
1027 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1028 ; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1029 ; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
1030 ; GFX90A-TGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc
1031 ; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1032 ; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
1033 ; GFX90A-TGSPLIT-NEXT: s_endpgm
1034 ;
1035 ; GFX940-NOTTGSPLIT-LABEL: private_nontemporal_volatile_load:
1036 ; GFX940-NOTTGSPLIT: ; %bb.0: ; %entry
1037 ; GFX940-NOTTGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1038 ; GFX940-NOTTGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1039 ; GFX940-NOTTGSPLIT-NEXT: s_nop 0
1040 ; GFX940-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1041 ; GFX940-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1042 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1043 ; GFX940-NOTTGSPLIT-NEXT: scratch_load_dword v1, off, s2 sc0 sc1
1044 ; GFX940-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0)
1045 ; GFX940-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1046 ; GFX940-NOTTGSPLIT-NEXT: s_endpgm
1047 ;
1048 ; GFX940-TGSPLIT-LABEL: private_nontemporal_volatile_load:
1049 ; GFX940-TGSPLIT: ; %bb.0: ; %entry
1050 ; GFX940-TGSPLIT-NEXT: s_mov_b64 s[0:1], s[2:3]
1051 ; GFX940-TGSPLIT-NEXT: s_load_dword s2, s[0:1], 0x0
1052 ; GFX940-TGSPLIT-NEXT: s_nop 0
1053 ; GFX940-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x8
1054 ; GFX940-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
1055 ; GFX940-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
1056 ; GFX940-TGSPLIT-NEXT: scratch_load_dword v1, off, s2 sc0 sc1
1057 ; GFX940-TGSPLIT-NEXT: s_waitcnt vmcnt(0)
1058 ; GFX940-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
1059 ; GFX940-TGSPLIT-NEXT: s_endpgm
1060 ;
1061 ; GFX11-WGP-LABEL: private_nontemporal_volatile_load:
1062 ; GFX11-WGP: ; %bb.0: ; %entry
1063 ; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1064 ; GFX11-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1065 ; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1066 ; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
1067 ; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
1068 ; GFX11-WGP-NEXT: scratch_load_b32 v1, off, s2 glc dlc
1069 ; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)
1070 ; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
1071 ; GFX11-WGP-NEXT: s_endpgm
1072 ;
1073 ; GFX11-CU-LABEL: private_nontemporal_volatile_load:
1074 ; GFX11-CU: ; %bb.0: ; %entry
1075 ; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1076 ; GFX11-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1077 ; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1078 ; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
1079 ; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
1080 ; GFX11-CU-NEXT: scratch_load_b32 v1, off, s2 glc dlc
1081 ; GFX11-CU-NEXT: s_waitcnt vmcnt(0)
1082 ; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1083 ; GFX11-CU-NEXT: s_endpgm
1084 ;
1085 ; GFX12-WGP-LABEL: private_nontemporal_volatile_load:
1086 ; GFX12-WGP: ; %bb.0: ; %entry
1087 ; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[2:3]
1088 ; GFX12-WGP-NEXT: s_load_b32 s2, s[0:1], 0x0
1089 ; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1090 ; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
1091 ; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
1092 ; GFX12-WGP-NEXT: scratch_load_b32 v1, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
1093 ; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0
1094 ; GFX12-WGP-NEXT: s_wait_samplecnt 0x0
1095 ; GFX12-WGP-NEXT: s_wait_loadcnt 0x0
1096 ; GFX12-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
1097 ; GFX12-WGP-NEXT: s_endpgm
1098 ;
1099 ; GFX12-CU-LABEL: private_nontemporal_volatile_load:
1100 ; GFX12-CU: ; %bb.0: ; %entry
1101 ; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[2:3]
1102 ; GFX12-CU-NEXT: s_load_b32 s2, s[0:1], 0x0
1103 ; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8
1104 ; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
1105 ; GFX12-CU-NEXT: s_wait_kmcnt 0x0
1106 ; GFX12-CU-NEXT: scratch_load_b32 v1, off, s2 th:TH_LOAD_NT scope:SCOPE_SYS
1107 ; GFX12-CU-NEXT: s_wait_bvhcnt 0x0
1108 ; GFX12-CU-NEXT: s_wait_samplecnt 0x0
1109 ; GFX12-CU-NEXT: s_wait_loadcnt 0x0
1110 ; GFX12-CU-NEXT: global_store_b32 v0, v1, s[0:1]
1111 ; GFX12-CU-NEXT: s_endpgm
1112 ptr addrspace(5) %in, ptr addrspace(1) %out) {
1113 entry:
1114 %val = load volatile i32, ptr addrspace(5) %in, align 4, !nontemporal !0
1115 store i32 %val, ptr addrspace(1) %out
1116 ret void
1117 }
1119 !0 = !{i32 1}
1120 declare i32 @llvm.amdgcn.workitem.id.x()