1 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
2 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi -amdgpu-enable-global-sgpr-addr < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
4 declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
5 declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
6 declare void @llvm.amdgcn.s.barrier() #1
7 declare i32 @llvm.amdgcn.workitem.id.x() #2
10 @stored_lds_ptr = addrspace(3) global i32 addrspace(3)* undef, align 4
11 @stored_constant_ptr = addrspace(3) global i32 addrspace(4)* undef, align 8
12 @stored_global_ptr = addrspace(3) global i32 addrspace(1)* undef, align 8
14 ; GCN-LABEL: {{^}}reorder_local_load_global_store_local_load:
15 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3
16 ; CI: buffer_store_dword
18 ; GFX9: global_store_dword
19 ; GFX9: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3
20 ; GFX9: global_store_dword
21 define amdgpu_kernel void @reorder_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
22 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
24 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
25 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3
27 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
28 store i32 99, i32 addrspace(1)* %gptr, align 4
29 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
31 %add = add nsw i32 %tmp1, %tmp2
33 store i32 %add, i32 addrspace(1)* %out, align 4
37 ; GCN-LABEL: {{^}}no_reorder_local_load_volatile_global_store_local_load:
38 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
39 ; CI: buffer_store_dword
40 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
42 ; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
43 ; GFX9: global_store_dword
44 ; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
45 define amdgpu_kernel void @no_reorder_local_load_volatile_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
46 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
48 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
49 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3
51 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
52 store volatile i32 99, i32 addrspace(1)* %gptr, align 4
53 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
55 %add = add nsw i32 %tmp1, %tmp2
57 store i32 %add, i32 addrspace(1)* %out, align 4
61 ; GCN-LABEL: {{^}}no_reorder_barrier_local_load_global_store_local_load:
62 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
63 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
64 ; CI: buffer_store_dword
66 ; GFX9-DAG: global_store_dword
67 ; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
69 ; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
70 ; GFX9-DAG: global_store_dword
71 define amdgpu_kernel void @no_reorder_barrier_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
72 %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
74 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
75 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3
77 %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
78 store i32 99, i32 addrspace(1)* %gptr, align 4
79 call void @llvm.amdgcn.s.barrier() #1
80 %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
82 %add = add nsw i32 %tmp1, %tmp2
84 store i32 %add, i32 addrspace(1)* %out, align 4
88 ; GCN-LABEL: {{^}}reorder_constant_load_global_store_constant_load:
89 ; GCN-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
90 ; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
91 ; CI: buffer_store_dword
93 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1
94 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3
96 ; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4
97 ; GFX9: global_store_dword
98 ; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc
100 ; CI: buffer_store_dword
101 ; GFX9: global_store_dword
102 define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
103 %ptr0 = load i32 addrspace(4)*, i32 addrspace(4)* addrspace(3)* @stored_constant_ptr, align 8
105 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1
106 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 3
108 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4
109 store i32 99, i32 addrspace(1)* %gptr, align 4
110 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4
112 %add = add nsw i32 %tmp1, %tmp2
114 store i32 %add, i32 addrspace(1)* %out, align 4
118 ; GCN-LABEL: {{^}}reorder_constant_load_local_store_constant_load:
119 ; GCN: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
120 ; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
122 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1
123 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3
125 ; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4
126 ; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc
129 ; CI: buffer_store_dword
130 ; GFX9: global_store_dword
131 define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr) #0 {
132 %ptr0 = load i32 addrspace(4)*, i32 addrspace(4)* addrspace(3)* @stored_constant_ptr, align 8
134 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1
135 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 3
137 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4
138 store i32 99, i32 addrspace(3)* %lptr, align 4
139 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4
141 %add = add nsw i32 %tmp1, %tmp2
143 store i32 %add, i32 addrspace(1)* %out, align 4
147 ; GCN-LABEL: {{^}}reorder_smrd_load_local_store_smrd_load:
152 ; CI: buffer_store_dword
153 ; GFX9: global_store_dword
154 define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(i32 addrspace(1)* %out, i32 addrspace(3)* noalias %lptr, i32 addrspace(4)* %ptr0) #0 {
155 %ptr1 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 1
156 %ptr2 = getelementptr inbounds i32, i32 addrspace(4)* %ptr0, i64 2
158 %tmp1 = load i32, i32 addrspace(4)* %ptr1, align 4
159 store i32 99, i32 addrspace(3)* %lptr, align 4
160 %tmp2 = load i32, i32 addrspace(4)* %ptr2, align 4
162 %add = add nsw i32 %tmp1, %tmp2
164 store i32 %add, i32 addrspace(1)* %out, align 4
168 ; GCN-LABEL: {{^}}reorder_global_load_local_store_global_load:
170 ; CI: buffer_load_dword
171 ; CI: buffer_load_dword
172 ; CI: buffer_store_dword
174 ; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4
175 ; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:12
177 define amdgpu_kernel void @reorder_global_load_local_store_global_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr, i32 addrspace(1)* %ptr0) #0 {
178 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 1
179 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 3
181 %tmp1 = load i32, i32 addrspace(1)* %ptr1, align 4
182 store i32 99, i32 addrspace(3)* %lptr, align 4
183 %tmp2 = load i32, i32 addrspace(1)* %ptr2, align 4
185 %add = add nsw i32 %tmp1, %tmp2
187 store i32 %add, i32 addrspace(1)* %out, align 4
191 ; GCN-LABEL: {{^}}reorder_local_offsets:
192 ; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:100 offset1:102
193 ; GCN-DAG: ds_write2_b32 {{v[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:3 offset1:100
194 ; GCN-DAG: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:408
195 ; CI: buffer_store_dword
196 ; GFX9: global_store_dword
198 define amdgpu_kernel void @reorder_local_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(3)* noalias nocapture %ptr0) #0 {
199 %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3
200 %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 100
201 %ptr3 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 102
203 store i32 123, i32 addrspace(3)* %ptr1, align 4
204 %tmp1 = load i32, i32 addrspace(3)* %ptr2, align 4
205 %tmp2 = load i32, i32 addrspace(3)* %ptr3, align 4
206 store i32 123, i32 addrspace(3)* %ptr2, align 4
207 %tmp3 = load i32, i32 addrspace(3)* %ptr1, align 4
208 store i32 789, i32 addrspace(3)* %ptr3, align 4
210 %add.0 = add nsw i32 %tmp2, %tmp1
211 %add.1 = add nsw i32 %add.0, %tmp3
212 store i32 %add.1, i32 addrspace(1)* %out, align 4
216 ; GCN-LABEL: {{^}}reorder_global_offsets:
217 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400
218 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408
219 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12
220 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400
221 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408
222 ; CI: buffer_store_dword
225 ; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:400
226 ; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:408
227 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:12
228 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:400
229 ; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:408
230 ; GFX9: global_store_dword
232 define amdgpu_kernel void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(1)* noalias nocapture %ptr0) #0 {
233 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3
234 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 100
235 %ptr3 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 102
237 store i32 123, i32 addrspace(1)* %ptr1, align 4
238 %tmp1 = load i32, i32 addrspace(1)* %ptr2, align 4
239 %tmp2 = load i32, i32 addrspace(1)* %ptr3, align 4
240 store i32 123, i32 addrspace(1)* %ptr2, align 4
241 %tmp3 = load i32, i32 addrspace(1)* %ptr1, align 4
242 store i32 789, i32 addrspace(1)* %ptr3, align 4
244 %add.0 = add nsw i32 %tmp2, %tmp1
245 %add.1 = add nsw i32 %add.0, %tmp3
246 store i32 %add.1, i32 addrspace(1)* %out, align 4
250 ; GCN-LABEL: {{^}}reorder_global_offsets_addr64_soffset0:
251 ; CI: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
252 ; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:28{{$}}
253 ; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:44{{$}}
258 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
263 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20{{$}}
265 ; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:36{{$}}
266 ; CI-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:52{{$}}
268 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}{{$}}
269 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:20
270 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} offset:12
271 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} offset:28
272 ; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} offset:44
274 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:36
275 ; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:52
277 define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(i32 addrspace(1)* noalias nocapture %ptr.base) #0 {
278 %id = call i32 @llvm.amdgcn.workitem.id.x()
279 %id.ext = sext i32 %id to i64
281 %ptr0 = getelementptr inbounds i32, i32 addrspace(1)* %ptr.base, i64 %id.ext
282 %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3
283 %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 5
284 %ptr3 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 7
285 %ptr4 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 9
286 %ptr5 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 11
287 %ptr6 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 13
289 store i32 789, i32 addrspace(1)* %ptr0, align 4
290 %tmp1 = load i32, i32 addrspace(1)* %ptr1, align 4
291 store i32 123, i32 addrspace(1)* %ptr2, align 4
292 %tmp2 = load i32, i32 addrspace(1)* %ptr3, align 4
293 %add.0 = add nsw i32 %tmp1, %tmp2
294 store i32 %add.0, i32 addrspace(1)* %ptr4, align 4
295 %tmp3 = load i32, i32 addrspace(1)* %ptr5, align 4
296 %add.1 = add nsw i32 %add.0, %tmp3
297 store i32 %add.1, i32 addrspace(1)* %ptr6, align 4
301 ; XGCN-LABEL: {{^}}reorder_local_load_tbuffer_store_local_load:
302 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x4
303 ; XCI: TBUFFER_STORE_FORMAT
304 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x8
305 ; define amdgpu_vs void @reorder_local_load_tbuffer_store_local_load(i32 addrspace(1)* %out, i32 %a1, i32 %vaddr) #0 {
306 ; %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
308 ; %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 1
309 ; %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 2
311 ; %tmp1 = load i32, i32 addrspace(3)* %ptr1, align 4
313 ; %vdata = insertelement <4 x i32> undef, i32 %a1, i32 0
314 ; call void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32> %vdata, <4 x i32> undef,
315 ; i32 %vaddr, i32 0, i32 0, i32 32, i32 14, i32 4, i1 1, i1 1)
317 ; %tmp2 = load i32, i32 addrspace(3)* %ptr2, align 4
319 ; %add = add nsw i32 %tmp1, %tmp2
321 ; store i32 %add, i32 addrspace(1)* %out, align 4
325 attributes #0 = { nounwind }
326 attributes #1 = { nounwind convergent }
327 attributes #2 = { nounwind readnone }