| blob | a0cab6b8d714064153b9044ffa71f05f8e7970c1 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -march=amdgcn -mcpu=gfx902 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
3 ; RUN: llc -march=amdgcn -mcpu=gfx1030 -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefix=GCN-SCRATCH %s
5 define amdgpu_kernel void @vector_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
6 ; GCN-LABEL: vector_clause:
7 ; GCN: ; %bb.0: ; %bb
8 ; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
9 ; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c
10 ; GCN-NEXT: v_lshlrev_b32_e32 v16, 4, v0
11 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
12 ; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[2:3]
13 ; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:16
14 ; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[2:3] offset:32
15 ; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[2:3] offset:48
16 ; GCN-NEXT: s_waitcnt vmcnt(3)
17 ; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5]
18 ; GCN-NEXT: s_waitcnt vmcnt(3)
19 ; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5] offset:16
20 ; GCN-NEXT: s_waitcnt vmcnt(3)
21 ; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[4:5] offset:32
22 ; GCN-NEXT: s_waitcnt vmcnt(3)
23 ; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[4:5] offset:48
24 ; GCN-NEXT: s_endpgm
25 ;
26 ; GCN-SCRATCH-LABEL: vector_clause:
27 ; GCN-SCRATCH: ; %bb.0: ; %bb
28 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
29 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v16, 4, v0
30 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2c
31 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
32 ; GCN-SCRATCH-NEXT: s_clause 0x3
33 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v16, s[2:3]
34 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:16
35 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[8:11], v16, s[2:3] offset:32
36 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[12:15], v16, s[2:3] offset:48
37 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)
38 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1]
39 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)
40 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16
41 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
42 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32
43 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
44 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48
45 ; GCN-SCRATCH-NEXT: s_endpgm
46 bb:
47 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
48 %tmp2 = zext i32 %tmp to i64
49 %tmp3 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp2
50 %tmp4 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp3, align 16
51 %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp2
52 %tmp6 = add nuw nsw i64 %tmp2, 1
53 %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp6
54 %tmp8 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp7, align 16
55 %tmp9 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp6
56 %tmp10 = add nuw nsw i64 %tmp2, 2
57 %tmp11 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp10
58 %tmp12 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp11, align 16
59 %tmp13 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp10
60 %tmp14 = add nuw nsw i64 %tmp2, 3
61 %tmp15 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 %tmp14
62 %tmp16 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp15, align 16
63 %tmp17 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 %tmp14
64 store <4 x i32> %tmp4, <4 x i32> addrspace(1)* %tmp5, align 16
65 store <4 x i32> %tmp8, <4 x i32> addrspace(1)* %tmp9, align 16
66 store <4 x i32> %tmp12, <4 x i32> addrspace(1)* %tmp13, align 16
67 store <4 x i32> %tmp16, <4 x i32> addrspace(1)* %tmp17, align 16
68 ret void
69 }
71 define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) {
72 ; GCN-LABEL: scalar_clause:
73 ; GCN: ; %bb.0: ; %bb
74 ; GCN-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x24
75 ; GCN-NEXT: s_load_dwordx2 s[18:19], s[0:1], 0x2c
76 ; GCN-NEXT: v_mov_b32_e32 v12, 0
77 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
78 ; GCN-NEXT: s_load_dwordx4 s[0:3], s[16:17], 0x0
79 ; GCN-NEXT: s_load_dwordx4 s[4:7], s[16:17], 0x10
80 ; GCN-NEXT: s_load_dwordx4 s[8:11], s[16:17], 0x20
81 ; GCN-NEXT: s_load_dwordx4 s[12:15], s[16:17], 0x30
82 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
83 ; GCN-NEXT: v_mov_b32_e32 v0, s0
84 ; GCN-NEXT: v_mov_b32_e32 v4, s4
85 ; GCN-NEXT: v_mov_b32_e32 v8, s8
86 ; GCN-NEXT: v_mov_b32_e32 v1, s1
87 ; GCN-NEXT: v_mov_b32_e32 v2, s2
88 ; GCN-NEXT: v_mov_b32_e32 v3, s3
89 ; GCN-NEXT: v_mov_b32_e32 v5, s5
90 ; GCN-NEXT: v_mov_b32_e32 v6, s6
91 ; GCN-NEXT: v_mov_b32_e32 v7, s7
92 ; GCN-NEXT: v_mov_b32_e32 v9, s9
93 ; GCN-NEXT: v_mov_b32_e32 v10, s10
94 ; GCN-NEXT: v_mov_b32_e32 v11, s11
95 ; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[18:19]
96 ; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[18:19] offset:16
97 ; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[18:19] offset:32
98 ; GCN-NEXT: v_mov_b32_e32 v0, s12
99 ; GCN-NEXT: v_mov_b32_e32 v1, s13
100 ; GCN-NEXT: v_mov_b32_e32 v2, s14
101 ; GCN-NEXT: v_mov_b32_e32 v3, s15
102 ; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[18:19] offset:48
103 ; GCN-NEXT: s_endpgm
104 ;
105 ; GCN-SCRATCH-LABEL: scalar_clause:
106 ; GCN-SCRATCH: ; %bb.0: ; %bb
107 ; GCN-SCRATCH-NEXT: s_clause 0x1
108 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x24
109 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x2c
110 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v16, 0
111 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
112 ; GCN-SCRATCH-NEXT: s_clause 0x3
113 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[12:13], 0x0
114 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[12:13], 0x10
115 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x20
116 ; GCN-SCRATCH-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x30
117 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
118 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s0
119 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v4, s4
120 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s1
121 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v2, s2
122 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v3, s3
123 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, s8
124 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, s5
125 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v6, s6
126 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v7, s7
127 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v12, s12
128 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v9, s9
129 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v10, s10
130 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v11, s11
131 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v13, s13
132 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v14, s14
133 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v15, s15
134 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
135 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
136 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
137 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
138 ; GCN-SCRATCH-NEXT: s_endpgm
139 bb:
140 %tmp = load <4 x i32>, <4 x i32> addrspace(1)* %arg, align 16
141 %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 1
142 %tmp3 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp2, align 16
143 %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 1
144 %tmp5 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 2
145 %tmp6 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp5, align 16
146 %tmp7 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 2
147 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg, i64 3
148 %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16
149 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg1, i64 3
150 store <4 x i32> %tmp, <4 x i32> addrspace(1)* %arg1, align 16
151 store <4 x i32> %tmp3, <4 x i32> addrspace(1)* %tmp4, align 16
152 store <4 x i32> %tmp6, <4 x i32> addrspace(1)* %tmp7, align 16
153 store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16
154 ret void
155 }
157 define void @mubuf_clause(<4 x i32> addrspace(5)* noalias nocapture readonly %arg, <4 x i32> addrspace(5)* noalias nocapture %arg1) {
158 ; GCN-LABEL: mubuf_clause:
159 ; GCN: ; %bb.0: ; %bb
160 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
161 ; GCN-NEXT: v_and_b32_e32 v2, 0x3ff, v2
162 ; GCN-NEXT: v_lshlrev_b32_e32 v2, 4, v2
163 ; GCN-NEXT: v_add_u32_e32 v0, v0, v2
164 ; GCN-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen
165 ; GCN-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen offset:4
166 ; GCN-NEXT: buffer_load_dword v5, v0, s[0:3], 0 offen offset:8
167 ; GCN-NEXT: buffer_load_dword v6, v0, s[0:3], 0 offen offset:12
168 ; GCN-NEXT: buffer_load_dword v7, v0, s[0:3], 0 offen offset:16
169 ; GCN-NEXT: buffer_load_dword v8, v0, s[0:3], 0 offen offset:20
170 ; GCN-NEXT: buffer_load_dword v9, v0, s[0:3], 0 offen offset:24
171 ; GCN-NEXT: buffer_load_dword v10, v0, s[0:3], 0 offen offset:28
172 ; GCN-NEXT: buffer_load_dword v11, v0, s[0:3], 0 offen offset:32
173 ; GCN-NEXT: buffer_load_dword v12, v0, s[0:3], 0 offen offset:36
174 ; GCN-NEXT: buffer_load_dword v13, v0, s[0:3], 0 offen offset:40
175 ; GCN-NEXT: buffer_load_dword v14, v0, s[0:3], 0 offen offset:44
176 ; GCN-NEXT: buffer_load_dword v15, v0, s[0:3], 0 offen offset:48
177 ; GCN-NEXT: buffer_load_dword v16, v0, s[0:3], 0 offen offset:52
178 ; GCN-NEXT: buffer_load_dword v17, v0, s[0:3], 0 offen offset:56
179 ; GCN-NEXT: s_nop 0
180 ; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:60
181 ; GCN-NEXT: v_add_u32_e32 v1, v1, v2
182 ; GCN-NEXT: s_waitcnt vmcnt(12)
183 ; GCN-NEXT: buffer_store_dword v6, v1, s[0:3], 0 offen offset:12
184 ; GCN-NEXT: buffer_store_dword v5, v1, s[0:3], 0 offen offset:8
185 ; GCN-NEXT: buffer_store_dword v4, v1, s[0:3], 0 offen offset:4
186 ; GCN-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
187 ; GCN-NEXT: s_waitcnt vmcnt(12)
188 ; GCN-NEXT: buffer_store_dword v10, v1, s[0:3], 0 offen offset:28
189 ; GCN-NEXT: buffer_store_dword v9, v1, s[0:3], 0 offen offset:24
190 ; GCN-NEXT: buffer_store_dword v8, v1, s[0:3], 0 offen offset:20
191 ; GCN-NEXT: buffer_store_dword v7, v1, s[0:3], 0 offen offset:16
192 ; GCN-NEXT: s_waitcnt vmcnt(12)
193 ; GCN-NEXT: buffer_store_dword v14, v1, s[0:3], 0 offen offset:44
194 ; GCN-NEXT: buffer_store_dword v13, v1, s[0:3], 0 offen offset:40
195 ; GCN-NEXT: buffer_store_dword v12, v1, s[0:3], 0 offen offset:36
196 ; GCN-NEXT: buffer_store_dword v11, v1, s[0:3], 0 offen offset:32
197 ; GCN-NEXT: s_waitcnt vmcnt(12)
198 ; GCN-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen offset:60
199 ; GCN-NEXT: buffer_store_dword v17, v1, s[0:3], 0 offen offset:56
200 ; GCN-NEXT: buffer_store_dword v16, v1, s[0:3], 0 offen offset:52
201 ; GCN-NEXT: buffer_store_dword v15, v1, s[0:3], 0 offen offset:48
202 ; GCN-NEXT: s_waitcnt vmcnt(0)
203 ; GCN-NEXT: s_setpc_b64 s[30:31]
204 ;
205 ; GCN-SCRATCH-LABEL: mubuf_clause:
206 ; GCN-SCRATCH: ; %bb.0: ; %bb
207 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
208 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
209 ; GCN-SCRATCH-NEXT: v_and_b32_e32 v2, 0x3ff, v2
210 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v18, 4, v2
211 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v0, v18
212 ; GCN-SCRATCH-NEXT: s_clause 0x3
213 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[2:5], v0, off
214 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[6:9], v0, off offset:16
215 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[10:13], v0, off offset:32
216 ; GCN-SCRATCH-NEXT: scratch_load_dwordx4 v[14:17], v0, off offset:48
217 ; GCN-SCRATCH-NEXT: v_add_nc_u32_e32 v0, v1, v18
218 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(3)
219 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[2:5], off
220 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(2)
221 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16
222 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
223 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32
224 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
225 ; GCN-SCRATCH-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48
226 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
227 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
228 bb:
229 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
230 %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp
231 %tmp3 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp2, align 16
232 %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp
233 %tmp5 = add nuw nsw i32 %tmp, 1
234 %tmp6 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp5
235 %tmp7 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp6, align 16
236 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp5
237 %tmp9 = add nuw nsw i32 %tmp, 2
238 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp9
239 %tmp11 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp10, align 16
240 %tmp12 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp9
241 %tmp13 = add nuw nsw i32 %tmp, 3
242 %tmp14 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp13
243 %tmp15 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp14, align 16
244 %tmp16 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp13
245 store <4 x i32> %tmp3, <4 x i32> addrspace(5)* %tmp4, align 16
246 store <4 x i32> %tmp7, <4 x i32> addrspace(5)* %tmp8, align 16
247 store <4 x i32> %tmp11, <4 x i32> addrspace(5)* %tmp12, align 16
248 store <4 x i32> %tmp15, <4 x i32> addrspace(5)* %tmp16, align 16
249 ret void
250 }
252 define amdgpu_kernel void @vector_clause_indirect(i64 addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture readnone %arg1, <4 x i32> addrspace(1)* noalias nocapture %arg2) {
253 ; GCN-LABEL: vector_clause_indirect:
254 ; GCN: ; %bb.0: ; %bb
255 ; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
256 ; GCN-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
257 ; GCN-NEXT: v_lshlrev_b32_e32 v0, 3, v0
258 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
259 ; GCN-NEXT: global_load_dwordx2 v[8:9], v0, s[2:3]
260 ; GCN-NEXT: s_waitcnt vmcnt(0)
261 ; GCN-NEXT: global_load_dwordx4 v[0:3], v[8:9], off
262 ; GCN-NEXT: global_load_dwordx4 v[4:7], v[8:9], off offset:16
263 ; GCN-NEXT: v_mov_b32_e32 v8, 0
264 ; GCN-NEXT: s_waitcnt vmcnt(1)
265 ; GCN-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
266 ; GCN-NEXT: s_waitcnt vmcnt(1)
267 ; GCN-NEXT: global_store_dwordx4 v8, v[4:7], s[4:5] offset:16
268 ; GCN-NEXT: s_endpgm
269 ;
270 ; GCN-SCRATCH-LABEL: vector_clause_indirect:
271 ; GCN-SCRATCH: ; %bb.0: ; %bb
272 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
273 ; GCN-SCRATCH-NEXT: v_lshlrev_b32_e32 v0, 3, v0
274 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
275 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v8, 0
276 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
277 ; GCN-SCRATCH-NEXT: global_load_dwordx2 v[4:5], v0, s[2:3]
278 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
279 ; GCN-SCRATCH-NEXT: s_clause 0x1
280 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[0:3], v[4:5], off
281 ; GCN-SCRATCH-NEXT: global_load_dwordx4 v[4:7], v[4:5], off offset:16
282 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
283 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
284 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
285 ; GCN-SCRATCH-NEXT: global_store_dwordx4 v8, v[4:7], s[0:1] offset:16
286 ; GCN-SCRATCH-NEXT: s_endpgm
287 bb:
288 %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
289 %tmp3 = zext i32 %tmp to i64
290 %tmp4 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 %tmp3
291 %tmp5 = bitcast i64 addrspace(1)* %tmp4 to <4 x i32> addrspace(1)* addrspace(1)*
292 %tmp6 = load <4 x i32> addrspace(1)*, <4 x i32> addrspace(1)* addrspace(1)* %tmp5, align 8
293 %tmp7 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp6, align 16
294 %tmp8 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %tmp6, i64 1
295 %tmp9 = load <4 x i32>, <4 x i32> addrspace(1)* %tmp8, align 16
296 store <4 x i32> %tmp7, <4 x i32> addrspace(1)* %arg2, align 16
297 %tmp10 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %arg2, i64 1
298 store <4 x i32> %tmp9, <4 x i32> addrspace(1)* %tmp10, align 16
299 ret void
300 }
302 define void @load_global_d16_hi(i16 addrspace(1)* %in, i16 %reg, <2 x i16> addrspace(1)* %out) {
303 ; GCN-LABEL: load_global_d16_hi:
304 ; GCN: ; %bb.0: ; %entry
305 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
306 ; GCN-NEXT: v_mov_b32_e32 v5, v2
307 ; GCN-NEXT: global_load_short_d16_hi v5, v[0:1], off
308 ; GCN-NEXT: s_nop 0
309 ; GCN-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64
310 ; GCN-NEXT: s_waitcnt vmcnt(1)
311 ; GCN-NEXT: global_store_dword v[3:4], v5, off
312 ; GCN-NEXT: s_waitcnt vmcnt(1)
313 ; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128
314 ; GCN-NEXT: s_waitcnt vmcnt(0)
315 ; GCN-NEXT: s_setpc_b64 s[30:31]
316 ;
317 ; GCN-SCRATCH-LABEL: load_global_d16_hi:
318 ; GCN-SCRATCH: ; %bb.0: ; %entry
319 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
320 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
321 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2
322 ; GCN-SCRATCH-NEXT: s_clause 0x1
323 ; GCN-SCRATCH-NEXT: global_load_short_d16_hi v5, v[0:1], off
324 ; GCN-SCRATCH-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:64
325 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
326 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off
327 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
328 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128
329 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
330 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
331 entry:
332 %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
333 %load1 = load i16, i16 addrspace(1)* %in
334 %load2 = load i16, i16 addrspace(1)* %gep
335 %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
336 %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
337 store <2 x i16> %build1, <2 x i16> addrspace(1)* %out
338 %build2 = insertelement <2 x i16> undef, i16 %reg, i32 0
339 %build3 = insertelement <2 x i16> %build2, i16 %load2, i32 1
340 %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32
341 store <2 x i16> %build3, <2 x i16> addrspace(1)* %gep2
342 ret void
343 }
345 define void @load_global_d16_lo(i16 addrspace(1)* %in, i32 %reg, <2 x i16> addrspace(1)* %out) {
346 ; GCN-LABEL: load_global_d16_lo:
347 ; GCN: ; %bb.0: ; %entry
348 ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
349 ; GCN-NEXT: v_mov_b32_e32 v5, v2
350 ; GCN-NEXT: global_load_short_d16 v5, v[0:1], off
351 ; GCN-NEXT: s_nop 0
352 ; GCN-NEXT: global_load_short_d16 v2, v[0:1], off offset:64
353 ; GCN-NEXT: s_waitcnt vmcnt(1)
354 ; GCN-NEXT: global_store_dword v[3:4], v5, off
355 ; GCN-NEXT: s_waitcnt vmcnt(1)
356 ; GCN-NEXT: global_store_dword v[3:4], v2, off offset:128
357 ; GCN-NEXT: s_waitcnt vmcnt(0)
358 ; GCN-NEXT: s_setpc_b64 s[30:31]
359 ;
360 ; GCN-SCRATCH-LABEL: load_global_d16_lo:
361 ; GCN-SCRATCH: ; %bb.0: ; %entry
362 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
363 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
364 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v5, v2
365 ; GCN-SCRATCH-NEXT: s_clause 0x1
366 ; GCN-SCRATCH-NEXT: global_load_short_d16 v5, v[0:1], off
367 ; GCN-SCRATCH-NEXT: global_load_short_d16 v2, v[0:1], off offset:64
368 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(1)
369 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v5, off
370 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
371 ; GCN-SCRATCH-NEXT: global_store_dword v[3:4], v2, off offset:128
372 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
373 ; GCN-SCRATCH-NEXT: s_setpc_b64 s[30:31]
374 entry:
375 %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 32
376 %reg.bc1 = bitcast i32 %reg to <2 x i16>
377 %reg.bc2 = bitcast i32 %reg to <2 x i16>
378 %load1 = load i16, i16 addrspace(1)* %in
379 %load2 = load i16, i16 addrspace(1)* %gep
380 %build1 = insertelement <2 x i16> %reg.bc1, i16 %load1, i32 0
381 %build2 = insertelement <2 x i16> %reg.bc2, i16 %load2, i32 0
382 %gep2 = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 32
383 store <2 x i16> %build1, <2 x i16> addrspace(1)* %out
384 store <2 x i16> %build2, <2 x i16> addrspace(1)* %gep2
385 ret void
386 }
388 define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc) {
389 ; GCN-LABEL: flat_scratch_load:
390 ; GCN: ; %bb.0: ; %.entry
391 ; GCN-NEXT: s_mov_b32 s16, SCRATCH_RSRC_DWORD0
392 ; GCN-NEXT: s_mov_b32 s17, SCRATCH_RSRC_DWORD1
393 ; GCN-NEXT: s_mov_b32 s18, -1
394 ; GCN-NEXT: s_mov_b32 s19, 0xe00000
395 ; GCN-NEXT: s_add_u32 s16, s16, s3
396 ; GCN-NEXT: s_addc_u32 s17, s17, 0
397 ; GCN-NEXT: s_mov_b64 s[12:13], exec
398 ; GCN-NEXT: s_wqm_b64 exec, exec
399 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000
400 ; GCN-NEXT: s_load_dwordx2 s[14:15], s[0:1], 0x24
401 ; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x44
402 ; GCN-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4
403 ; GCN-NEXT: s_waitcnt vmcnt(0)
404 ; GCN-NEXT: ;;#ASMSTART
405 ; GCN-NEXT: ;;#ASMEND
406 ; GCN-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:4
407 ; GCN-NEXT: s_brev_b32 s0, 1
408 ; GCN-NEXT: s_waitcnt lgkmcnt(0)
409 ; GCN-NEXT: v_mov_b32_e32 v0, s14
410 ; GCN-NEXT: s_mov_b32 s3, 0
411 ; GCN-NEXT: s_mov_b32 s1, s0
412 ; GCN-NEXT: s_mov_b32 s2, s0
413 ; GCN-NEXT: v_mov_b32_e32 v1, s15
414 ; GCN-NEXT: s_and_b64 exec, exec, s[12:13]
415 ; GCN-NEXT: image_sample v0, v[0:1], s[4:11], s[0:3] dmask:0x1
416 ; GCN-NEXT: s_waitcnt vmcnt(0)
417 ; GCN-NEXT: v_add_f32_e32 v0, v2, v0
418 ; GCN-NEXT: exp mrt0 v0, off, off, off done vm
419 ; GCN-NEXT: s_endpgm
420 ;
421 ; GCN-SCRATCH-LABEL: flat_scratch_load:
422 ; GCN-SCRATCH: ; %bb.0: ; %.entry
423 ; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5
424 ; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0
425 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
426 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
427 ; GCN-SCRATCH-NEXT: s_mov_b32 s9, exec_lo
428 ; GCN-SCRATCH-NEXT: s_wqm_b32 exec_lo, exec_lo
429 ; GCN-SCRATCH-NEXT: s_clause 0x1
430 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[10:11], s[0:1], 0x24
431 ; GCN-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x44
432 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000
433 ; GCN-SCRATCH-NEXT: s_brev_b32 s8, 1
434 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4
435 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
436 ; GCN-SCRATCH-NEXT: ;;#ASMSTART
437 ; GCN-SCRATCH-NEXT: ;;#ASMEND
438 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
439 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s10
440 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s11
441 ; GCN-SCRATCH-NEXT: s_and_b32 exec_lo, exec_lo, s9
442 ; GCN-SCRATCH-NEXT: s_mov_b32 s11, 0
443 ; GCN-SCRATCH-NEXT: s_mov_b32 s9, s8
444 ; GCN-SCRATCH-NEXT: s_mov_b32 s10, s8
445 ; GCN-SCRATCH-NEXT: scratch_load_dword v2, off, off offset:4
446 ; GCN-SCRATCH-NEXT: image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
447 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
448 ; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v2, v0
449 ; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm
450 ; GCN-SCRATCH-NEXT: s_endpgm
451 .entry:
452 %alloca = alloca float, align 4, addrspace(5)
453 store volatile float 5.5, float addrspace(5)* %alloca
454 call void asm sideeffect "", ""()
455 ; There was a bug with flat scratch instructions that do not not use any address registers (ST mode).
456 ; To trigger, the scratch_load has to be immediately before the image_sample in MIR.
457 %load = load float, float addrspace(5)* %alloca
458 %val = call <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 9, float %a, float %b, <8 x i32> %desc, <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 0>, i1 false, i32 0, i32 0)
459 %val0 = extractelement <2 x float> %val, i32 0
460 %valadd = fadd float %load, %val0
461 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
462 ret void
463 }
465 define amdgpu_kernel void @flat_scratch_load_clause(float %a, float %b, <8 x i32> %desc) {
466 ; GCN-LABEL: flat_scratch_load_clause:
467 ; GCN: ; %bb.0: ; %.entry
468 ; GCN-NEXT: s_mov_b32 s4, SCRATCH_RSRC_DWORD0
469 ; GCN-NEXT: s_mov_b32 s5, SCRATCH_RSRC_DWORD1
470 ; GCN-NEXT: s_mov_b32 s6, -1
471 ; GCN-NEXT: s_mov_b32 s7, 0xe00000
472 ; GCN-NEXT: s_add_u32 s4, s4, s3
473 ; GCN-NEXT: s_addc_u32 s5, s5, 0
474 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000
475 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
476 ; GCN-NEXT: s_waitcnt vmcnt(0)
477 ; GCN-NEXT: v_mov_b32_e32 v0, 0x40d00000
478 ; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:8
479 ; GCN-NEXT: s_waitcnt vmcnt(0)
480 ; GCN-NEXT: ;;#ASMSTART
481 ; GCN-NEXT: ;;#ASMEND
482 ; GCN-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:4
483 ; GCN-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:8
484 ; GCN-NEXT: s_waitcnt vmcnt(0)
485 ; GCN-NEXT: v_add_f32_e32 v0, v0, v1
486 ; GCN-NEXT: exp mrt0 v0, off, off, off done vm
487 ; GCN-NEXT: s_endpgm
488 ;
489 ; GCN-SCRATCH-LABEL: flat_scratch_load_clause:
490 ; GCN-SCRATCH: ; %bb.0: ; %.entry
491 ; GCN-SCRATCH-NEXT: s_add_u32 s2, s2, s5
492 ; GCN-SCRATCH-NEXT: s_addc_u32 s3, s3, 0
493 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
494 ; GCN-SCRATCH-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
495 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, 0x40b00000
496 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, 0x40d00000
497 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v0, off offset:4
498 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
499 ; GCN-SCRATCH-NEXT: scratch_store_dword off, v1, off offset:8
500 ; GCN-SCRATCH-NEXT: s_waitcnt_vscnt null, 0x0
501 ; GCN-SCRATCH-NEXT: ;;#ASMSTART
502 ; GCN-SCRATCH-NEXT: ;;#ASMEND
503 ; GCN-SCRATCH-NEXT: s_clause 0x1
504 ; GCN-SCRATCH-NEXT: scratch_load_dword v0, off, off offset:4
505 ; GCN-SCRATCH-NEXT: scratch_load_dword v1, off, off offset:8
506 ; GCN-SCRATCH-NEXT: s_waitcnt vmcnt(0)
507 ; GCN-SCRATCH-NEXT: v_add_f32_e32 v0, v0, v1
508 ; GCN-SCRATCH-NEXT: exp mrt0 v0, off, off, off done vm
509 ; GCN-SCRATCH-NEXT: s_endpgm
510 .entry:
511 %alloca = alloca float, align 4, addrspace(5)
512 %alloca2 = alloca float, align 4, addrspace(5)
513 store volatile float 5.5, float addrspace(5)* %alloca
514 store volatile float 6.5, float addrspace(5)* %alloca2
515 call void asm sideeffect "", ""()
516 %load0 = load float, float addrspace(5)* %alloca
517 %load1 = load float, float addrspace(5)* %alloca2
518 %valadd = fadd float %load0, %load1
519 call void @llvm.amdgcn.exp.f32(i32 immarg 0, i32 immarg 1, float %valadd, float undef, float undef, float undef, i1 immarg true, i1 immarg true)
520 ret void
521 }
523 declare i32 @llvm.amdgcn.workitem.id.x()
524 declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)
525 declare <2 x float> @llvm.amdgcn.image.sample.2d.v2f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)