| blob | 6f419ab2cc67ebaa99a2d1edd667ab5a08bdfce3 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -march=amdgcn -mcpu=gfx902 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
3 ; RUN: llc -march=amdgcn -mcpu=gfx1030 -mattr=+enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-SCRATCH %s
5 define amdgpu_kernel void @vector_clause(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture %arg1) {
6 ; GCN-LABEL: vector_clause:
7 ; GCN: ; %bb.0: ; %bb
8 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
9 ; GCN-NEXT: v_lshlrev_b32_e32 v16, 4, v0
10 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
11 ; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
12 ; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
13 ; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
14 ; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
15 ; GCN-NEXT: s_waitcnt vmcnt(3)
16 ; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
17 ; GCN-NEXT: s_waitcnt vmcnt(3)
18 ; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:16
19 ; GCN-NEXT: s_waitcnt vmcnt(3)
20 ; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:32
21 ; GCN-NEXT: s_waitcnt vmcnt(3)
22 ; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:48
23 ; GCN-NEXT: s_endpgm
24 ;
25 ; GCN-SCRATCH-LABEL: vector_clause:
26 ; GCN-SCRATCH: ; %bb.0: ; %bb
27 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
28 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v16, 4, v0
29 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
30 ; GCN-SCRATCH-NEXT: s_clause 0x3
31 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
32 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
33 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
34 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
35 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)
36 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
37 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)
38 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:16
39 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
40 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:32
41 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
42 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:48
43 ; GCN-SCRATCH-NEXT: s_endpgm
44 bb:
45 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
46 %tmp2 = zext i32 %tmp to i64
47 %tmp3 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp2
48 %tmp4 = load <4 x i32>, ptr addrspace(1) %tmp3, align 16
49 %tmp5 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp2
50 %tmp6 = add nuw nsw i64 %tmp2, 1
51 %tmp7 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp6
52 %tmp8 = load <4 x i32>, ptr addrspace(1) %tmp7, align 16
53 %tmp9 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp6
54 %tmp10 = add nuw nsw i64 %tmp2, 2
55 %tmp11 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp10
56 %tmp12 = load <4 x i32>, ptr addrspace(1) %tmp11, align 16
57 %tmp13 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp10
58 %tmp14 = add nuw nsw i64 %tmp2, 3
59 %tmp15 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 %tmp14
60 %tmp16 = load <4 x i32>, ptr addrspace(1) %tmp15, align 16
61 %tmp17 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 %tmp14
62 store <4 x i32> %tmp4, ptr addrspace(1) %tmp5, align 16
63 store <4 x i32> %tmp8, ptr addrspace(1) %tmp9, align 16
64 store <4 x i32> %tmp12, ptr addrspace(1) %tmp13, align 16
65 store <4 x i32> %tmp16, ptr addrspace(1) %tmp17, align 16
66 ret void
67 }
69 define amdgpu_kernel void @scalar_clause(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture %arg1) {
70 ; GCN-LABEL: scalar_clause:
71 ; GCN: ; %bb.0: ; %bb
72 ; GCN-NEXT: s_load_dwordx4 s[16:19], s[0:1], 0x24
73 ; GCN-NEXT: v_mov_b32_e32 v16, 0
74 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
75 ; GCN-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
76 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
77 ; GCN-NEXT: v_mov_b32_e32 v0, s0
78 ; GCN-NEXT: v_mov_b32_e32 v1, s1
79 ; GCN-NEXT: v_mov_b32_e32 v2, s2
80 ; GCN-NEXT: v_mov_b32_e32 v3, s3
81 ; GCN-NEXT: v_mov_b32_e32 v4, s4
82 ; GCN-NEXT: v_mov_b32_e32 v5, s5
83 ; GCN-NEXT: v_mov_b32_e32 v6, s6
84 ; GCN-NEXT: v_mov_b32_e32 v7, s7
85 ; GCN-NEXT: v_mov_b32_e32 v8, s8
86 ; GCN-NEXT: v_mov_b32_e32 v9, s9
87 ; GCN-NEXT: v_mov_b32_e32 v10, s10
88 ; GCN-NEXT: v_mov_b32_e32 v11, s11
89 ; GCN-NEXT: v_mov_b32_e32 v12, s12
90 ; GCN-NEXT: v_mov_b32_e32 v13, s13
91 ; GCN-NEXT: v_mov_b32_e32 v14, s14
92 ; GCN-NEXT: v_mov_b32_e32 v15, s15
93 ; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[18:19]
94 ; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[18:19] offset:16
95 ; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[18:19] offset:32
96 ; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[18:19] offset:48
97 ; GCN-NEXT: s_endpgm
98 ;
99 ; GCN-SCRATCH-LABEL: scalar_clause:
100 ; GCN-SCRATCH: ; %bb.0: ; %bb
101 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[16:19], s[0:1], 0x24
102 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v16, 0
103 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
104 ; GCN-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
105 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
106 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s0
107 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s1
108 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v2, s2
109 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v3, s3
110 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v4, s4
111 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, s5
112 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v6, s6
113 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v7, s7
114 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, s8
115 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v9, s9
116 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v10, s10
117 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v11, s11
118 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v12, s12
119 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v13, s13
120 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v14, s14
121 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v15, s15
122 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[18:19]
123 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[18:19] offset:16
124 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[18:19] offset:32
125 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[18:19] offset:48
126 ; GCN-SCRATCH-NEXT: s_endpgm
127 bb:
128 %tmp = load <4 x i32>, ptr addrspace(1) %arg, align 16
129 %tmp2 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 1
130 %tmp3 = load <4 x i32>, ptr addrspace(1) %tmp2, align 16
131 %tmp4 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 1
132 %tmp5 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 2
133 %tmp6 = load <4 x i32>, ptr addrspace(1) %tmp5, align 16
134 %tmp7 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 2
135 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg, i64 3
136 %tmp9 = load <4 x i32>, ptr addrspace(1) %tmp8, align 16
137 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg1, i64 3
138 store <4 x i32> %tmp, ptr addrspace(1) %arg1, align 16
139 store <4 x i32> %tmp3, ptr addrspace(1) %tmp4, align 16
140 store <4 x i32> %tmp6, ptr addrspace(1) %tmp7, align 16
141 store <4 x i32> %tmp9, ptr addrspace(1) %tmp10, align 16
142 ret void
143 }
145 define void @mubuf_clause(ptr addrspace(5) noalias nocapture readonly %arg, ptr addrspace(5) noalias nocapture %arg1) {
146 ; GCN-LABEL: mubuf_clause:
147 ; GCN: ; %bb.0: ; %bb
148 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
149 ; GCN-NEXT: v_lshlrev_b32_e32 v2, 4, v31
150 ; GCN-NEXT: v_and_b32_e32 v2, 0x3ff0, v2
151 ; GCN-NEXT: v_add_u32_e32 v0, v0, v2
152 ; GCN-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen offset:12
153 ; GCN-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:8
154 ; GCN-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:4
155 ; GCN-NEXT: buffer_load_dword v6, v0, s[0:3], 0 offen
156 ; GCN-NEXT: buffer_load_dword v7, v0, s[0:3], 0 offen offset:28
157 ; GCN-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen offset:24
158 ; GCN-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen offset:20
159 ; GCN-NEXT: buffer_load_dword v10, v0, s[0:3], 0 offen offset:16
160 ; GCN-NEXT: buffer_load_dword v11, v0, s[0:3], 0 offen offset:44
161 ; GCN-NEXT: buffer_load_dword v12, v0, s[0:3], 0 offen offset:40
162 ; GCN-NEXT: buffer_load_dword v13, v0, s[0:3], 0 offen offset:36
163 ; GCN-NEXT: buffer_load_dword v14, v0, s[0:3], 0 offen offset:32
164 ; GCN-NEXT: buffer_load_dword v15, v0, s[0:3], 0 offen offset:60
165 ; GCN-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen offset:56
166 ; GCN-NEXT: buffer_load_dword v17, v0, s[0:3], 0 offen offset:52
167 ; GCN-NEXT: s_nop 0
168 ; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:48
169 ; GCN-NEXT: v_add_u32_e32 v1, v1, v2
170 ; GCN-NEXT: s_waitcnt vmcnt(15)
171 ; GCN-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen offset:12
172 ; GCN-NEXT: s_waitcnt vmcnt(15)
173 ; GCN-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen offset:8
174 ; GCN-NEXT: s_waitcnt vmcnt(15)
175 ; GCN-NEXT: buffer_store_dword v5, v1, s[0:3], 0 offen offset:4
176 ; GCN-NEXT: s_waitcnt vmcnt(15)
177 ; GCN-NEXT: buffer_store_dword v6, v1, s[0:3], 0 offen
178 ; GCN-NEXT: s_waitcnt vmcnt(15)
179 ; GCN-NEXT: buffer_store_dword v7, v1, s[0:3], 0 offen offset:28
180 ; GCN-NEXT: s_waitcnt vmcnt(15)
181 ; GCN-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen offset:24
182 ; GCN-NEXT: s_waitcnt vmcnt(15)
183 ; GCN-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen offset:20
184 ; GCN-NEXT: s_waitcnt vmcnt(15)
185 ; GCN-NEXT: buffer_store_dword v10, v1, s[0:3], 0 offen offset:16
186 ; GCN-NEXT: s_waitcnt vmcnt(15)
187 ; GCN-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen offset:44
188 ; GCN-NEXT: s_waitcnt vmcnt(15)
189 ; GCN-NEXT: buffer_store_dword v12, v1, s[0:3], 0 offen offset:40
190 ; GCN-NEXT: s_waitcnt vmcnt(15)
191 ; GCN-NEXT: buffer_store_dword v13, v1, s[0:3], 0 offen offset:36
192 ; GCN-NEXT: s_waitcnt vmcnt(15)
193 ; GCN-NEXT: buffer_store_dword v14, v1, s[0:3], 0 offen offset:32
194 ; GCN-NEXT: s_waitcnt vmcnt(15)
195 ; GCN-NEXT: buffer_store_dword v15, v1, s[0:3], 0 offen offset:60
196 ; GCN-NEXT: s_waitcnt vmcnt(15)
197 ; GCN-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen offset:56
198 ; GCN-NEXT: s_waitcnt vmcnt(15)
199 ; GCN-NEXT: buffer_store_dword v17, v1, s[0:3], 0 offen offset:52
200 ; GCN-NEXT: s_waitcnt vmcnt(15)
201 ; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:48
202 ; GCN-NEXT: s_waitcnt vmcnt(0)
203 ; GCN-NEXT: s_setpc_b64 s[30:31]
204 ;
205 ; GCN-SCRATCH-LABEL: mubuf_clause:
206 ; GCN-SCRATCH: ; %bb.0: ; %bb
207 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
208 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v2, 4, v31
209 ; GCN-SCRATCH-NEXT: v_and_b32_e32 v18, 0x3ff0, v2
210 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v0, v18
211 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v6, 16, v0
212 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v10, 32, v0
213 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v14, 48, v0
214 ; GCN-SCRATCH-NEXT: s_clause 0x3
215 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[2:5], v0, off
216 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[6:9], v6, off
217 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[10:13], v10, off
218 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[14:17], v14, off
219 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v1, v18
220 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v1, 16, v0
221 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v18, 32, v0
222 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v19, 48, v0
223 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)
224 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[2:5], off
225 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)
226 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v1, v[6:9], off
227 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
228 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v18, v[10:13], off
229 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
230 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v19, v[14:17], off
231 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
232 bb:
233 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
234 %tmp2 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg, i32 %tmp
235 %tmp3 = load <4 x i32>, ptr addrspace(5) %tmp2, align 16
236 %tmp4 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg1, i32 %tmp
237 %tmp5 = add nuw nsw i32 %tmp, 1
238 %tmp6 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg, i32 %tmp5
239 %tmp7 = load <4 x i32>, ptr addrspace(5) %tmp6, align 16
240 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg1, i32 %tmp5
241 %tmp9 = add nuw nsw i32 %tmp, 2
242 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg, i32 %tmp9
243 %tmp11 = load <4 x i32>, ptr addrspace(5) %tmp10, align 16
244 %tmp12 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg1, i32 %tmp9
245 %tmp13 = add nuw nsw i32 %tmp, 3
246 %tmp14 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg, i32 %tmp13
247 %tmp15 = load <4 x i32>, ptr addrspace(5) %tmp14, align 16
248 %tmp16 = getelementptr inbounds <4 x i32>, ptr addrspace(5) %arg1, i32 %tmp13
249 store <4 x i32> %tmp3, ptr addrspace(5) %tmp4, align 16
250 store <4 x i32> %tmp7, ptr addrspace(5) %tmp8, align 16
251 store <4 x i32> %tmp11, ptr addrspace(5) %tmp12, align 16
252 store <4 x i32> %tmp15, ptr addrspace(5) %tmp16, align 16
253 ret void
254 }
256 define amdgpu_kernel void @vector_clause_indirect(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture readnone %arg1, ptr addrspace(1) noalias nocapture %arg2) {
257 ; GCN-LABEL: vector_clause_indirect:
258 ; GCN: ; %bb.0: ; %bb
259 ; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
260 ; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
261 ; GCN-NEXT: v_lshlrev_b32_e32 v0, 3, v0
262 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
263 ; GCN-NEXT: global_load_dwordx2 v[8:9], v0, s[2:3]
264 ; GCN-NEXT: s_waitcnt vmcnt(0)
265 ; GCN-NEXT: global_load_dwordx4 v[0:3], v[8:9], off
266 ; GCN-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16
267 ; GCN-NEXT: v_mov_b32_e32 v8, 0
268 ; GCN-NEXT: s_waitcnt vmcnt(1)
269 ; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
270 ; GCN-NEXT: s_waitcnt vmcnt(1)
271 ; GCN-NEXT: global_store_dwordx4 v8, v[4:7], s[4:5] offset:16
272 ; GCN-NEXT: s_endpgm
273 ;
274 ; GCN-SCRATCH-LABEL: vector_clause_indirect:
275 ; GCN-SCRATCH: ; %bb.0: ; %bb
276 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
277 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v0, 3, v0
278 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
279 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, 0
280 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
281 ; GCN-SCRATCH-NEXT: global_load_dwordx2 v[4:5], v0, s[2:3]
282 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
283 ; GCN-SCRATCH-NEXT: s_clause 0x1
284 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v[4:5], off
285 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v[4:5], off offset:16
286 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
287 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
288 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
289 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
290 ; GCN-SCRATCH-NEXT: s_endpgm
291 bb:
292 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
293 %tmp3 = zext i32 %tmp to i64
294 %tmp4 = getelementptr inbounds i64, ptr addrspace(1) %arg, i64 %tmp3
295 %tmp6 = load ptr addrspace(1), ptr addrspace(1) %tmp4, align 8
296 %tmp7 = load <4 x i32>, ptr addrspace(1) %tmp6, align 16
297 %tmp8 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %tmp6, i64 1
298 %tmp9 = load <4 x i32>, ptr addrspace(1) %tmp8, align 16
299 store <4 x i32> %tmp7, ptr addrspace(1) %arg2, align 16
300 %tmp10 = getelementptr inbounds <4 x i32>, ptr addrspace(1) %arg2, i64 1
301 store <4 x i32> %tmp9, ptr addrspace(1) %tmp10, align 16
302 ret void
303 }
305 define void @load_global_d16_hi(ptr addrspace(1) %in, i16 %reg, ptr addrspace(1) %out) {
306 ; GCN-LABEL: load_global_d16_hi:
307 ; GCN: ; %bb.0: ; %entry
308 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
309 ; GCN-NEXT: v_mov_b32_e32 v5, v2
310 ; GCN-NEXT: global_load_short_d16_hi v5, v[0:1], off
311 ; GCN-NEXT: s_nop 0
312 ; GCN-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64
313 ; GCN-NEXT: s_waitcnt vmcnt(1)
314 ; GCN-NEXT: global_store_dword v[3:4], v5, off
315 ; GCN-NEXT: s_waitcnt vmcnt(1)
316 ; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128
317 ; GCN-NEXT: s_waitcnt vmcnt(0)
318 ; GCN-NEXT: s_setpc_b64 s[30:31]
319 ;
320 ; GCN-SCRATCH-LABEL: load_global_d16_hi:
321 ; GCN-SCRATCH: ; %bb.0: ; %entry
322 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
323 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2
324 ; GCN-SCRATCH-NEXT: s_clause 0x1
325 ; GCN-SCRATCH-NEXT: global_load_short_d16_hi v5, v[0:1], off
326 ; GCN-SCRATCH-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64
327 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
328 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off
329 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
330 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128
331 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
332 entry:
333 %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 32
334 %load1 = load i16, ptr addrspace(1) %in
335 %load2 = load i16, ptr addrspace(1) %gep
336 %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
337 %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
338 store <2 x i16> %build1, ptr addrspace(1) %out
339 %build2 = insertelement <2 x i16> undef, i16 %reg, i32 0
340 %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1
341 %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %out, i64 32
342 store <2 x i16> %build3, ptr addrspace(1) %gep2
343 ret void
344 }
346 define void @load_global_d16_lo(ptr addrspace(1) %in, i32 %reg, ptr addrspace(1) %out) {
347 ; GCN-LABEL: load_global_d16_lo:
348 ; GCN: ; %bb.0: ; %entry
349 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
350 ; GCN-NEXT: v_mov_b32_e32 v5, v2
351 ; GCN-NEXT: global_load_short_d16 v5, v[0:1], off
352 ; GCN-NEXT: s_nop 0
353 ; GCN-NEXT: global_load_short_d16 v2, v[0:1], off offset:64
354 ; GCN-NEXT: s_waitcnt vmcnt(1)
355 ; GCN-NEXT: global_store_dword v[3:4], v5, off
356 ; GCN-NEXT: s_waitcnt vmcnt(1)
357 ; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128
358 ; GCN-NEXT: s_waitcnt vmcnt(0)
359 ; GCN-NEXT: s_setpc_b64 s[30:31]
360 ;
361 ; GCN-SCRATCH-LABEL: load_global_d16_lo:
362 ; GCN-SCRATCH: ; %bb.0: ; %entry
363 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
364 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2
365 ; GCN-SCRATCH-NEXT: s_clause 0x1
366 ; GCN-SCRATCH-NEXT: global_load_short_d16 v5, v[0:1], off
367 ; GCN-SCRATCH-NEXT: global_load_short_d16 v2, v[0:1], off offset:64
368 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
369 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off
370 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
371 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128
372 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
373 entry:
374 %gep = getelementptr inbounds i16, ptr addrspace(1) %in, i64 32
375 %reg.bc1 = bitcast i32 %reg to <2 x i16>
376 %reg.bc2 = bitcast i32 %reg to <2 x i16>
377 %load1 = load i16, ptr addrspace(1) %in
378 %load2 = load i16, ptr addrspace(1) %gep
379 %build1 = insertelement <2 x i16> %reg.bc1, i16 %load1, i32 0
380 %build2 = insertelement <2 x i16> %reg.bc2, i16 %load2, i32 0
381 %gep2 = getelementptr inbounds <2 x i16>, ptr addrspace(1) %out, i64 32
382 store <2 x i16> %build1, ptr addrspace(1) %out
383 store <2 x i16> %build2, ptr addrspace(1) %gep2
384 ret void
385 }
387 define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc) {
388 ; GCN-LABEL: flat_scratch_load:
389 ; GCN: ; %bb.0: ; %.entry
390 ; GCN-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
391 ; GCN-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
392 ; GCN-NEXT: s_mov_b32 s18, -1
393 ; GCN-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x24
394 ; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x44
395 ; GCN-NEXT: s_mov_b32 s19, 0xe00000
396 ; GCN-NEXT: s_add_u32 s16, s16, s3
397 ; GCN-NEXT: s_addc_u32 s17, s17, 0
398 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000
399 ; GCN-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4
400 ; GCN-NEXT: s_waitcnt vmcnt(0)
401 ; GCN-NEXT: s_brev_b32 s0, 1
402 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
403 ; GCN-NEXT: v_mov_b32_e32 v0, s12
404 ; GCN-NEXT: s_mov_b32 s3, 0
405 ; GCN-NEXT: s_mov_b32 s1, s0
406 ; GCN-NEXT: s_mov_b32 s2, s0
407 ; GCN-NEXT: v_mov_b32_e32 v1, s13
408 ; GCN-NEXT: ;;#ASMSTART
409 ; GCN-NEXT: ;;#ASMEND
410 ; GCN-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:4
411 ; GCN-NEXT: s_nop 0
412 ; GCN-NEXT: image_sample v0, v[0:1], s[4:11], s[0:3] dmask:0x1
413 ; GCN-NEXT: s_waitcnt vmcnt(0)
414 ; GCN-NEXT: v_add_f32_e32 v0, v2, v0
415 ; GCN-NEXT: exp mrt0 v0, off, off, off done vm
416 ; GCN-NEXT: s_endpgm
417 ;
418 ; GCN-SCRATCH-LABEL: flat_scratch_load:
419 ; GCN-SCRATCH: ; %bb.0: ; %.entry
420 ; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5
421 ; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0
422 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
423 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
424 ; GCN-SCRATCH-NEXT: s_clause 0x1
425 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[10:11], s[0:1], 0x24
426 ; GCN-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x44
427 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000
428 ; GCN-SCRATCH-NEXT: s_brev_b32 s8, 1
429 ; GCN-SCRATCH-NEXT: s_mov_b32 s9, s8
430 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4
431 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
432 ; GCN-SCRATCH-NEXT: ;;#ASMSTART
433 ; GCN-SCRATCH-NEXT: ;;#ASMEND
434 ; GCN-SCRATCH-NEXT: scratch_load_dword v2, off, off offset:4
435 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
436 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s10
437 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s11
438 ; GCN-SCRATCH-NEXT: s_mov_b32 s11, 0
439 ; GCN-SCRATCH-NEXT: s_mov_b32 s10, s8
440 ; GCN-SCRATCH-NEXT: image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
441 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
442 ; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v2, v0
443 ; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm
444 ; GCN-SCRATCH-NEXT: s_endpgm
445 .entry:
446 %alloca = alloca float, align 4, addrspace(5)
447 store volatile float 5.5, ptr addrspace(5) %alloca
448 call void asm sideeffect "", ""()
449 ; There was a bug with flat scratch instructions that do not not use any address registers (ST mode).
450 ; To trigger, the scratch_load has to be immediately before the image_sample in MIR.
451 %load = load float, ptr addrspace(5) %alloca
452 %val = call <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 9, float %a, float %b, <8 x i32> %desc, <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 0>, i1 false, i32 0, i32 0)
453 %val0 = extractelement <2 x float> %val, i32 0
454 %valadd = fadd float %load, %val0
455 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
456 ret void
457 }
459 define amdgpu_kernel void @flat_scratch_load_clause(float %a, float %b, <8 x i32> %desc) {
460 ; GCN-LABEL: flat_scratch_load_clause:
461 ; GCN: ; %bb.0: ; %.entry
462 ; GCN-NEXT: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
463 ; GCN-NEXT: s_mov_b32 s5, SCRATCH_RSRC_DWORD1
464 ; GCN-NEXT: s_mov_b32 s6, -1
465 ; GCN-NEXT: s_mov_b32 s7, 0xe00000
466 ; GCN-NEXT: s_add_u32 s4, s4, s3
467 ; GCN-NEXT: s_addc_u32 s5, s5, 0
468 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000
469 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
470 ; GCN-NEXT: s_waitcnt vmcnt(0)
471 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40d00000
472 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:8
473 ; GCN-NEXT: s_waitcnt vmcnt(0)
474 ; GCN-NEXT: ;;#ASMSTART
475 ; GCN-NEXT: ;;#ASMEND
476 ; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:4
477 ; GCN-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:8
478 ; GCN-NEXT: s_waitcnt vmcnt(0)
479 ; GCN-NEXT: v_add_f32_e32 v0, v0, v1
480 ; GCN-NEXT: exp mrt0 v0, off, off, off done vm
481 ; GCN-NEXT: s_endpgm
482 ;
483 ; GCN-SCRATCH-LABEL: flat_scratch_load_clause:
484 ; GCN-SCRATCH: ; %bb.0: ; %.entry
485 ; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5
486 ; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0
487 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
488 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
489 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000
490 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, 0x40d00000
491 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4
492 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
493 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v1, off offset:8
494 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
495 ; GCN-SCRATCH-NEXT: ;;#ASMSTART
496 ; GCN-SCRATCH-NEXT: ;;#ASMEND
497 ; GCN-SCRATCH-NEXT: s_clause 0x1
498 ; GCN-SCRATCH-NEXT: scratch_load_dword v0, off, off offset:4
499 ; GCN-SCRATCH-NEXT: scratch_load_dword v1, off, off offset:8
500 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
501 ; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v0, v1
502 ; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm
503 ; GCN-SCRATCH-NEXT: s_endpgm
504 .entry:
505 %alloca = alloca float, align 4, addrspace(5)
506 %alloca2 = alloca float, align 4, addrspace(5)
507 store volatile float 5.5, ptr addrspace(5) %alloca
508 store volatile float 6.5, ptr addrspace(5) %alloca2
509 call void asm sideeffect "", ""()
510 %load0 = load float, ptr addrspace(5) %alloca
511 %load1 = load float, ptr addrspace(5) %alloca2
512 %valadd = fadd float %load0, %load1
513 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
514 ret void
515 }
517 declare i32 @llvm.amdgcn.workitem.id.x()
518 declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)
519 declare <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)