| blob | 5275a8190807cb093912cbf5cd385002fa56ba2b |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
3 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
4 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
6 define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
7 ; GFX9-LABEL: s_lshr_v2i16:
8 ; GFX9: ; %bb.0:
9 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
10 ; GFX9-NEXT: s_load_dword s4, s[0:1], 0x2c
11 ; GFX9-NEXT: s_load_dword s0, s[0:1], 0x30
12 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
13 ; GFX9-NEXT: v_mov_b32_e32 v0, s2
14 ; GFX9-NEXT: v_mov_b32_e32 v2, s4
15 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
16 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, s0, v2
17 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
18 ; GFX9-NEXT: s_endpgm
19 ;
20 ; VI-LABEL: s_lshr_v2i16:
21 ; VI: ; %bb.0:
22 ; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
23 ; VI-NEXT: s_load_dword s5, s[0:1], 0x2c
24 ; VI-NEXT: s_load_dword s0, s[0:1], 0x30
25 ; VI-NEXT: s_mov_b32 s4, 0xffff
26 ; VI-NEXT: s_waitcnt lgkmcnt(0)
27 ; VI-NEXT: s_and_b32 s1, s5, s4
28 ; VI-NEXT: s_and_b32 s4, s0, s4
29 ; VI-NEXT: s_lshr_b32 s5, s5, 16
30 ; VI-NEXT: s_lshr_b32 s0, s0, 16
31 ; VI-NEXT: s_lshr_b32 s0, s5, s0
32 ; VI-NEXT: v_mov_b32_e32 v0, s4
33 ; VI-NEXT: v_bfe_u32 v0, s1, v0, 16
34 ; VI-NEXT: s_lshl_b32 s0, s0, 16
35 ; VI-NEXT: v_or_b32_e32 v2, s0, v0
36 ; VI-NEXT: v_mov_b32_e32 v0, s2
37 ; VI-NEXT: v_mov_b32_e32 v1, s3
38 ; VI-NEXT: flat_store_dword v[0:1], v2
39 ; VI-NEXT: s_endpgm
40 ;
41 ; CI-LABEL: s_lshr_v2i16:
42 ; CI: ; %bb.0:
43 ; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
44 ; CI-NEXT: s_load_dword s2, s[0:1], 0xb
45 ; CI-NEXT: s_load_dword s0, s[0:1], 0xc
46 ; CI-NEXT: s_mov_b32 s3, 0xffff
47 ; CI-NEXT: s_mov_b32 s7, 0xf000
48 ; CI-NEXT: s_mov_b32 s6, -1
49 ; CI-NEXT: s_waitcnt lgkmcnt(0)
50 ; CI-NEXT: s_lshr_b32 s1, s2, 16
51 ; CI-NEXT: s_lshr_b32 s8, s0, 16
52 ; CI-NEXT: s_and_b32 s0, s0, s3
53 ; CI-NEXT: v_mov_b32_e32 v0, s0
54 ; CI-NEXT: s_lshr_b32 s0, s1, s8
55 ; CI-NEXT: s_and_b32 s2, s2, s3
56 ; CI-NEXT: v_bfe_u32 v0, s2, v0, 16
57 ; CI-NEXT: s_lshl_b32 s0, s0, 16
58 ; CI-NEXT: v_or_b32_e32 v0, s0, v0
59 ; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
60 ; CI-NEXT: s_endpgm
61 %result = lshr <2 x i16> %lhs, %rhs
62 store <2 x i16> %result, <2 x i16> addrspace(1)* %out
63 ret void
64 }
66 define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
67 ; GFX9-LABEL: v_lshr_v2i16:
68 ; GFX9: ; %bb.0:
69 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
70 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
71 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
72 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
73 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v2
74 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
75 ; GFX9-NEXT: global_load_dword v3, v[0:1], off
76 ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4
77 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2
78 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
79 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
80 ; GFX9-NEXT: s_waitcnt vmcnt(0)
81 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, v4, v3
82 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
83 ; GFX9-NEXT: s_endpgm
84 ;
85 ; VI-LABEL: v_lshr_v2i16:
86 ; VI: ; %bb.0:
87 ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
88 ; VI-NEXT: v_lshlrev_b32_e32 v4, 2, v0
89 ; VI-NEXT: s_waitcnt lgkmcnt(0)
90 ; VI-NEXT: v_mov_b32_e32 v1, s3
91 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4
92 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
93 ; VI-NEXT: v_add_u32_e32 v2, vcc, 4, v0
94 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
95 ; VI-NEXT: flat_load_dword v5, v[0:1]
96 ; VI-NEXT: flat_load_dword v2, v[2:3]
97 ; VI-NEXT: v_mov_b32_e32 v1, s1
98 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4
99 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
100 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
101 ; VI-NEXT: v_lshrrev_b16_e32 v3, v2, v5
102 ; VI-NEXT: v_lshrrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
103 ; VI-NEXT: v_or_b32_e32 v2, v3, v2
104 ; VI-NEXT: flat_store_dword v[0:1], v2
105 ; VI-NEXT: s_endpgm
106 ;
107 ; CI-LABEL: v_lshr_v2i16:
108 ; CI: ; %bb.0:
109 ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
110 ; CI-NEXT: s_mov_b32 s7, 0xf000
111 ; CI-NEXT: s_mov_b32 s6, 0
112 ; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
113 ; CI-NEXT: v_mov_b32_e32 v1, 0
114 ; CI-NEXT: s_waitcnt lgkmcnt(0)
115 ; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
116 ; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
117 ; CI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4
118 ; CI-NEXT: s_mov_b32 s8, 0xffff
119 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7]
120 ; CI-NEXT: s_waitcnt vmcnt(1)
121 ; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v2
122 ; CI-NEXT: s_waitcnt vmcnt(0)
123 ; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v3
124 ; CI-NEXT: v_and_b32_e32 v2, s8, v2
125 ; CI-NEXT: v_and_b32_e32 v3, s8, v3
126 ; CI-NEXT: v_bfe_u32 v2, v2, v3, 16
127 ; CI-NEXT: v_lshrrev_b32_e32 v3, v5, v4
128 ; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
129 ; CI-NEXT: v_or_b32_e32 v2, v2, v3
130 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
131 ; CI-NEXT: s_endpgm
132 %tid = call i32 @llvm.amdgcn.workitem.id.x()
133 %tid.ext = sext i32 %tid to i64
134 %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
135 %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
136 %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in.gep, i32 1
137 %a = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
138 %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr
139 %result = lshr <2 x i16> %a, %b
140 store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
141 ret void
142 }
144 define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
145 ; GFX9-LABEL: lshr_v_s_v2i16:
146 ; GFX9: ; %bb.0:
147 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
148 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
149 ; GFX9-NEXT: s_load_dword s0, s[0:1], 0x34
150 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
151 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
152 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s6, v2
153 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
154 ; GFX9-NEXT: global_load_dword v3, v[0:1], off
155 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
156 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
157 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
158 ; GFX9-NEXT: s_waitcnt vmcnt(0)
159 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, s0, v3
160 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
161 ; GFX9-NEXT: s_endpgm
162 ;
163 ; VI-LABEL: lshr_v_s_v2i16:
164 ; VI: ; %bb.0:
165 ; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
166 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
167 ; VI-NEXT: s_load_dword s0, s[0:1], 0x34
168 ; VI-NEXT: s_waitcnt lgkmcnt(0)
169 ; VI-NEXT: v_mov_b32_e32 v1, s7
170 ; VI-NEXT: v_add_u32_e32 v0, vcc, s6, v2
171 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
172 ; VI-NEXT: flat_load_dword v3, v[0:1]
173 ; VI-NEXT: s_lshr_b32 s1, s0, 16
174 ; VI-NEXT: v_mov_b32_e32 v4, s1
175 ; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v2
176 ; VI-NEXT: v_mov_b32_e32 v1, s5
177 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
178 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
179 ; VI-NEXT: v_lshrrev_b16_e32 v2, s0, v3
180 ; VI-NEXT: v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
181 ; VI-NEXT: v_or_b32_e32 v2, v2, v3
182 ; VI-NEXT: flat_store_dword v[0:1], v2
183 ; VI-NEXT: s_endpgm
184 ;
185 ; CI-LABEL: lshr_v_s_v2i16:
186 ; CI: ; %bb.0:
187 ; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
188 ; CI-NEXT: s_load_dword s8, s[0:1], 0xd
189 ; CI-NEXT: s_mov_b32 s3, 0xf000
190 ; CI-NEXT: s_mov_b32 s2, 0
191 ; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
192 ; CI-NEXT: s_waitcnt lgkmcnt(0)
193 ; CI-NEXT: s_mov_b64 s[0:1], s[6:7]
194 ; CI-NEXT: v_mov_b32_e32 v1, 0
195 ; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
196 ; CI-NEXT: s_lshr_b32 s9, s8, 16
197 ; CI-NEXT: s_mov_b32 s10, 0xffff
198 ; CI-NEXT: s_and_b32 s8, s8, s10
199 ; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
200 ; CI-NEXT: s_waitcnt vmcnt(0)
201 ; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
202 ; CI-NEXT: v_and_b32_e32 v2, s10, v2
203 ; CI-NEXT: v_lshrrev_b32_e32 v3, s9, v3
204 ; CI-NEXT: v_bfe_u32 v2, v2, s8, 16
205 ; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
206 ; CI-NEXT: v_or_b32_e32 v2, v2, v3
207 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
208 ; CI-NEXT: s_endpgm
209 %tid = call i32 @llvm.amdgcn.workitem.id.x()
210 %tid.ext = sext i32 %tid to i64
211 %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
212 %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
213 %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
214 %result = lshr <2 x i16> %vgpr, %sgpr
215 store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
216 ret void
217 }
219 define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
220 ; GFX9-LABEL: lshr_s_v_v2i16:
221 ; GFX9: ; %bb.0:
222 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
223 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
224 ; GFX9-NEXT: s_load_dword s0, s[0:1], 0x34
225 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
226 ; GFX9-NEXT: v_mov_b32_e32 v1, s7
227 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s6, v2
228 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
229 ; GFX9-NEXT: global_load_dword v3, v[0:1], off
230 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s4, v2
231 ; GFX9-NEXT: v_mov_b32_e32 v1, s5
232 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
233 ; GFX9-NEXT: s_waitcnt vmcnt(0)
234 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, v3, s0
235 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
236 ; GFX9-NEXT: s_endpgm
237 ;
238 ; VI-LABEL: lshr_s_v_v2i16:
239 ; VI: ; %bb.0:
240 ; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
241 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
242 ; VI-NEXT: s_load_dword s0, s[0:1], 0x34
243 ; VI-NEXT: s_waitcnt lgkmcnt(0)
244 ; VI-NEXT: v_mov_b32_e32 v1, s7
245 ; VI-NEXT: v_add_u32_e32 v0, vcc, s6, v2
246 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
247 ; VI-NEXT: flat_load_dword v3, v[0:1]
248 ; VI-NEXT: s_lshr_b32 s1, s0, 16
249 ; VI-NEXT: v_mov_b32_e32 v4, s1
250 ; VI-NEXT: v_add_u32_e32 v0, vcc, s4, v2
251 ; VI-NEXT: v_mov_b32_e32 v1, s5
252 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
253 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
254 ; VI-NEXT: v_lshrrev_b16_e64 v2, v3, s0
255 ; VI-NEXT: v_lshrrev_b16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
256 ; VI-NEXT: v_or_b32_e32 v2, v2, v3
257 ; VI-NEXT: flat_store_dword v[0:1], v2
258 ; VI-NEXT: s_endpgm
259 ;
260 ; CI-LABEL: lshr_s_v_v2i16:
261 ; CI: ; %bb.0:
262 ; CI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
263 ; CI-NEXT: s_load_dword s8, s[0:1], 0xd
264 ; CI-NEXT: s_mov_b32 s3, 0xf000
265 ; CI-NEXT: s_mov_b32 s2, 0
266 ; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
267 ; CI-NEXT: s_waitcnt lgkmcnt(0)
268 ; CI-NEXT: s_mov_b64 s[0:1], s[6:7]
269 ; CI-NEXT: v_mov_b32_e32 v1, 0
270 ; CI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
271 ; CI-NEXT: s_lshr_b32 s9, s8, 16
272 ; CI-NEXT: s_mov_b32 s10, 0xffff
273 ; CI-NEXT: s_and_b32 s8, s8, s10
274 ; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
275 ; CI-NEXT: s_waitcnt vmcnt(0)
276 ; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
277 ; CI-NEXT: v_and_b32_e32 v2, s10, v2
278 ; CI-NEXT: v_lshr_b32_e32 v3, s9, v3
279 ; CI-NEXT: v_bfe_u32 v2, s8, v2, 16
280 ; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
281 ; CI-NEXT: v_or_b32_e32 v2, v2, v3
282 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
283 ; CI-NEXT: s_endpgm
284 %tid = call i32 @llvm.amdgcn.workitem.id.x()
285 %tid.ext = sext i32 %tid to i64
286 %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
287 %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
288 %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
289 %result = lshr <2 x i16> %sgpr, %vgpr
290 store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
291 ret void
292 }
294 define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
295 ; GFX9-LABEL: lshr_imm_v_v2i16:
296 ; GFX9: ; %bb.0:
297 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
298 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
299 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
300 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
301 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v2
302 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
303 ; GFX9-NEXT: global_load_dword v3, v[0:1], off
304 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2
305 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
306 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
307 ; GFX9-NEXT: s_waitcnt vmcnt(0)
308 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, v3, 8 op_sel_hi:[1,0]
309 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
310 ; GFX9-NEXT: s_endpgm
311 ;
312 ; VI-LABEL: lshr_imm_v_v2i16:
313 ; VI: ; %bb.0:
314 ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
315 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
316 ; VI-NEXT: v_mov_b32_e32 v3, 8
317 ; VI-NEXT: s_waitcnt lgkmcnt(0)
318 ; VI-NEXT: v_mov_b32_e32 v1, s3
319 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
320 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
321 ; VI-NEXT: flat_load_dword v4, v[0:1]
322 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
323 ; VI-NEXT: v_mov_b32_e32 v1, s1
324 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
325 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
326 ; VI-NEXT: v_lshrrev_b16_e64 v2, v4, 8
327 ; VI-NEXT: v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
328 ; VI-NEXT: v_or_b32_e32 v2, v2, v3
329 ; VI-NEXT: flat_store_dword v[0:1], v2
330 ; VI-NEXT: s_endpgm
331 ;
332 ; CI-LABEL: lshr_imm_v_v2i16:
333 ; CI: ; %bb.0:
334 ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
335 ; CI-NEXT: s_mov_b32 s7, 0xf000
336 ; CI-NEXT: s_mov_b32 s6, 0
337 ; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
338 ; CI-NEXT: v_mov_b32_e32 v1, 0
339 ; CI-NEXT: s_waitcnt lgkmcnt(0)
340 ; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
341 ; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
342 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7]
343 ; CI-NEXT: s_waitcnt vmcnt(0)
344 ; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
345 ; CI-NEXT: v_and_b32_e32 v2, 0xffff, v2
346 ; CI-NEXT: v_lshr_b32_e32 v3, 8, v3
347 ; CI-NEXT: v_bfe_u32 v2, 8, v2, 16
348 ; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
349 ; CI-NEXT: v_or_b32_e32 v2, v2, v3
350 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
351 ; CI-NEXT: s_endpgm
352 %tid = call i32 @llvm.amdgcn.workitem.id.x()
353 %tid.ext = sext i32 %tid to i64
354 %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
355 %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
356 %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
357 %result = lshr <2 x i16> <i16 8, i16 8>, %vgpr
358 store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
359 ret void
360 }
362 define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
363 ; GFX9-LABEL: lshr_v_imm_v2i16:
364 ; GFX9: ; %bb.0:
365 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
366 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
367 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
368 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
369 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v2
370 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
371 ; GFX9-NEXT: global_load_dword v3, v[0:1], off
372 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v2
373 ; GFX9-NEXT: v_mov_b32_e32 v1, s1
374 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
375 ; GFX9-NEXT: s_waitcnt vmcnt(0)
376 ; GFX9-NEXT: v_pk_lshrrev_b16 v2, 8, v3 op_sel_hi:[0,1]
377 ; GFX9-NEXT: global_store_dword v[0:1], v2, off
378 ; GFX9-NEXT: s_endpgm
379 ;
380 ; VI-LABEL: lshr_v_imm_v2i16:
381 ; VI: ; %bb.0:
382 ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
383 ; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
384 ; VI-NEXT: s_waitcnt lgkmcnt(0)
385 ; VI-NEXT: v_mov_b32_e32 v1, s3
386 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
387 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
388 ; VI-NEXT: flat_load_dword v3, v[0:1]
389 ; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
390 ; VI-NEXT: v_mov_b32_e32 v1, s1
391 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
392 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
393 ; VI-NEXT: v_lshrrev_b32_e32 v2, 24, v3
394 ; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
395 ; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
396 ; VI-NEXT: flat_store_dword v[0:1], v2
397 ; VI-NEXT: s_endpgm
398 ;
399 ; CI-LABEL: lshr_v_imm_v2i16:
400 ; CI: ; %bb.0:
401 ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
402 ; CI-NEXT: s_mov_b32 s7, 0xf000
403 ; CI-NEXT: s_mov_b32 s6, 0
404 ; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
405 ; CI-NEXT: v_mov_b32_e32 v1, 0
406 ; CI-NEXT: s_waitcnt lgkmcnt(0)
407 ; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
408 ; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
409 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7]
410 ; CI-NEXT: s_waitcnt vmcnt(0)
411 ; CI-NEXT: v_lshrrev_b32_e32 v2, 8, v2
412 ; CI-NEXT: v_and_b32_e32 v2, 0xff00ff, v2
413 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
414 ; CI-NEXT: s_endpgm
415 %tid = call i32 @llvm.amdgcn.workitem.id.x()
416 %tid.ext = sext i32 %tid to i64
417 %in.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i64 %tid.ext
418 %out.gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i64 %tid.ext
419 %vgpr = load <2 x i16>, <2 x i16> addrspace(1)* %in.gep
420 %result = lshr <2 x i16> %vgpr, <i16 8, i16 8>
421 store <2 x i16> %result, <2 x i16> addrspace(1)* %out.gep
422 ret void
423 }
425 define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
426 ; GFX9-LABEL: v_lshr_v4i16:
427 ; GFX9: ; %bb.0:
428 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
429 ; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
430 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
431 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
432 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v4
433 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
434 ; GFX9-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
435 ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off offset:8
436 ; GFX9-NEXT: v_mov_b32_e32 v5, s1
437 ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, s0, v4
438 ; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
439 ; GFX9-NEXT: s_waitcnt vmcnt(0)
440 ; GFX9-NEXT: v_pk_lshrrev_b16 v1, v1, v3
441 ; GFX9-NEXT: v_pk_lshrrev_b16 v0, v0, v2
442 ; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
443 ; GFX9-NEXT: s_endpgm
444 ;
445 ; VI-LABEL: v_lshr_v4i16:
446 ; VI: ; %bb.0:
447 ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
448 ; VI-NEXT: v_lshlrev_b32_e32 v4, 3, v0
449 ; VI-NEXT: s_waitcnt lgkmcnt(0)
450 ; VI-NEXT: v_mov_b32_e32 v1, s3
451 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4
452 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
453 ; VI-NEXT: v_add_u32_e32 v2, vcc, 8, v0
454 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
455 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
456 ; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
457 ; VI-NEXT: v_mov_b32_e32 v5, s1
458 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4
459 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
460 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
461 ; VI-NEXT: v_lshrrev_b16_e32 v6, v3, v1
462 ; VI-NEXT: v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
463 ; VI-NEXT: v_lshrrev_b16_e32 v3, v2, v0
464 ; VI-NEXT: v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
465 ; VI-NEXT: v_or_b32_e32 v1, v6, v1
466 ; VI-NEXT: v_or_b32_e32 v0, v3, v0
467 ; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
468 ; VI-NEXT: s_endpgm
469 ;
470 ; CI-LABEL: v_lshr_v4i16:
471 ; CI: ; %bb.0:
472 ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
473 ; CI-NEXT: s_mov_b32 s7, 0xf000
474 ; CI-NEXT: s_mov_b32 s6, 0
475 ; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
476 ; CI-NEXT: v_mov_b32_e32 v1, 0
477 ; CI-NEXT: s_waitcnt lgkmcnt(0)
478 ; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
479 ; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
480 ; CI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8
481 ; CI-NEXT: s_mov_b32 s8, 0xffff
482 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7]
483 ; CI-NEXT: s_waitcnt vmcnt(1)
484 ; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
485 ; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
486 ; CI-NEXT: s_waitcnt vmcnt(0)
487 ; CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4
488 ; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v5
489 ; CI-NEXT: v_and_b32_e32 v2, s8, v2
490 ; CI-NEXT: v_and_b32_e32 v4, s8, v4
491 ; CI-NEXT: v_and_b32_e32 v3, s8, v3
492 ; CI-NEXT: v_and_b32_e32 v5, s8, v5
493 ; CI-NEXT: v_bfe_u32 v3, v3, v5, 16
494 ; CI-NEXT: v_lshrrev_b32_e32 v5, v9, v7
495 ; CI-NEXT: v_bfe_u32 v2, v2, v4, 16
496 ; CI-NEXT: v_lshrrev_b32_e32 v4, v8, v6
497 ; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
498 ; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
499 ; CI-NEXT: v_or_b32_e32 v3, v3, v5
500 ; CI-NEXT: v_or_b32_e32 v2, v2, v4
501 ; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
502 ; CI-NEXT: s_endpgm
503 %tid = call i32 @llvm.amdgcn.workitem.id.x()
504 %tid.ext = sext i32 %tid to i64
505 %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
506 %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
507 %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in.gep, i32 1
508 %a = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
509 %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr
510 %result = lshr <4 x i16> %a, %b
511 store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
512 ret void
513 }
515 define amdgpu_kernel void @lshr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
516 ; GFX9-LABEL: lshr_v_imm_v4i16:
517 ; GFX9: ; %bb.0:
518 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
519 ; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
520 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
521 ; GFX9-NEXT: v_mov_b32_e32 v1, s3
522 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s2, v2
523 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
524 ; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
525 ; GFX9-NEXT: v_mov_b32_e32 v3, s1
526 ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
527 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
528 ; GFX9-NEXT: s_waitcnt vmcnt(0)
529 ; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
530 ; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
531 ; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
532 ; GFX9-NEXT: s_endpgm
533 ;
534 ; VI-LABEL: lshr_v_imm_v4i16:
535 ; VI: ; %bb.0:
536 ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
537 ; VI-NEXT: v_lshlrev_b32_e32 v2, 3, v0
538 ; VI-NEXT: s_waitcnt lgkmcnt(0)
539 ; VI-NEXT: v_mov_b32_e32 v1, s3
540 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
541 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
542 ; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
543 ; VI-NEXT: v_mov_b32_e32 v3, s1
544 ; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2
545 ; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
546 ; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
547 ; VI-NEXT: v_lshrrev_b32_e32 v4, 24, v1
548 ; VI-NEXT: v_lshrrev_b32_e32 v5, 24, v0
549 ; VI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
550 ; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
551 ; VI-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
552 ; VI-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
553 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
554 ; VI-NEXT: s_endpgm
555 ;
556 ; CI-LABEL: lshr_v_imm_v4i16:
557 ; CI: ; %bb.0:
558 ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
559 ; CI-NEXT: s_mov_b32 s7, 0xf000
560 ; CI-NEXT: s_mov_b32 s6, 0
561 ; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
562 ; CI-NEXT: v_mov_b32_e32 v1, 0
563 ; CI-NEXT: s_waitcnt lgkmcnt(0)
564 ; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
565 ; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
566 ; CI-NEXT: s_mov_b32 s8, 0xff00ff
567 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7]
568 ; CI-NEXT: s_waitcnt vmcnt(0)
569 ; CI-NEXT: v_lshrrev_b32_e32 v3, 8, v3
570 ; CI-NEXT: v_lshrrev_b32_e32 v2, 8, v2
571 ; CI-NEXT: v_and_b32_e32 v3, s8, v3
572 ; CI-NEXT: v_and_b32_e32 v2, s8, v2
573 ; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
574 ; CI-NEXT: s_endpgm
575 %tid = call i32 @llvm.amdgcn.workitem.id.x()
576 %tid.ext = sext i32 %tid to i64
577 %in.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i64 %tid.ext
578 %out.gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i64 %tid.ext
579 %vgpr = load <4 x i16>, <4 x i16> addrspace(1)* %in.gep
580 %result = lshr <4 x i16> %vgpr, <i16 8, i16 8, i16 8, i16 8>
581 store <4 x i16> %result, <4 x i16> addrspace(1)* %out.gep
582 ret void
583 }
585 declare i32 @llvm.amdgcn.workitem.id.x() #1
587 attributes #0 = { nounwind }
588 attributes #1 = { nounwind readnone }