| blob | ac9540bc0d842b690699767a6d9a6a0a73f3153d |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
4 ; Make sure that AMDGPUCodeGenPrepare introduces mul24 intrinsics
5 ; after SLSR, as the intrinsics would interfere. It's unclear if these
6 ; should be introduced before LSR or not. It seems to help in some
7 ; cases, and hurt others.
9 define void @lsr_order_mul24_0(i32 %arg, i32 %arg2, i32 %arg6, i32 %arg13, i32 %arg16) #0 {
10 ; GFX9-LABEL: lsr_order_mul24_0:
11 ; GFX9: ; %bb.0: ; %bb
12 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
13 ; GFX9-NEXT: global_load_dword v5, v[0:1], off
14 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v2
15 ; GFX9-NEXT: v_sub_u32_e32 v4, v4, v1
16 ; GFX9-NEXT: s_mov_b64 s[4:5], 0
17 ; GFX9-NEXT: s_waitcnt vmcnt(0)
18 ; GFX9-NEXT: ds_write_b32 v0, v5
19 ; GFX9-NEXT: .LBB0_1: ; %bb23
20 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
21 ; GFX9-NEXT: v_mul_u32_u24_e32 v5, v0, v2
22 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
23 ; GFX9-NEXT: v_sub_u32_e32 v5, v4, v5
24 ; GFX9-NEXT: v_add_u32_e32 v5, v5, v0
25 ; GFX9-NEXT: v_cmp_ge_u32_e32 vcc, v5, v3
26 ; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
27 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
28 ; GFX9-NEXT: s_cbranch_execnz .LBB0_1
29 ; GFX9-NEXT: ; %bb.2: ; %.loopexit
30 ; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
31 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
32 ; GFX9-NEXT: s_setpc_b64 s[30:31]
33 bb:
34 %tmp22 = and i32 %arg6, 16777215
35 br label %bb23
37 .loopexit: ; preds = %bb23
38 ret void
40 bb23: ; preds = %bb23, %bb
41 %tmp24 = phi i32 [ %arg, %bb ], [ %tmp47, %bb23 ]
42 %tmp28 = and i32 %tmp24, 16777215
43 %tmp29 = mul i32 %tmp28, %tmp22
44 %tmp30 = sub i32 %tmp24, %tmp29
45 %tmp31 = add i32 %tmp30, %arg16
46 %tmp37 = icmp ult i32 %tmp31, %arg13
47 %tmp44 = load float, float addrspace(1)* undef, align 4
48 store float %tmp44, float addrspace(3)* undef, align 4
49 %tmp47 = add i32 %tmp24, %arg2
50 br i1 %tmp37, label %bb23, label %.loopexit
51 }
53 define void @lsr_order_mul24_1(i32 %arg, i32 %arg1, i32 %arg2, float addrspace(3)* nocapture %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7, i32 %arg8, i32 %arg9, float addrspace(1)* nocapture readonly %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15, i32 %arg16, i1 zeroext %arg17, i1 zeroext %arg18) #0 {
54 ; GFX9-LABEL: lsr_order_mul24_1:
55 ; GFX9: ; %bb.0: ; %bb
56 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
57 ; GFX9-NEXT: v_and_b32_e32 v5, 1, v18
58 ; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
59 ; GFX9-NEXT: v_cmp_lt_u32_e64 s[4:5], v0, v1
60 ; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
61 ; GFX9-NEXT: s_cbranch_execz .LBB1_3
62 ; GFX9-NEXT: ; %bb.1: ; %bb19
63 ; GFX9-NEXT: v_cvt_f32_u32_e32 v7, v6
64 ; GFX9-NEXT: v_add_u32_e32 v4, v4, v0
65 ; GFX9-NEXT: v_and_b32_e32 v5, 0xffffff, v6
66 ; GFX9-NEXT: v_lshl_add_u32 v6, v4, 2, v3
67 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v7
68 ; GFX9-NEXT: v_lshlrev_b32_e32 v8, 2, v2
69 ; GFX9-NEXT: v_add_u32_e32 v9, v17, v12
70 ; GFX9-NEXT: s_mov_b64 s[10:11], 0
71 ; GFX9-NEXT: v_mov_b32_e32 v4, 0
72 ; GFX9-NEXT: ; implicit-def: $vgpr3
73 ; GFX9-NEXT: .LBB1_2: ; %bb23
74 ; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
75 ; GFX9-NEXT: v_cvt_f32_u32_e32 v3, v0
76 ; GFX9-NEXT: v_add_u32_e32 v12, v17, v0
77 ; GFX9-NEXT: v_madak_f32 v3, v3, v7, 0x3727c5ac
78 ; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3
79 ; GFX9-NEXT: v_mul_u32_u24_e32 v18, v3, v5
80 ; GFX9-NEXT: v_add_u32_e32 v19, v3, v16
81 ; GFX9-NEXT: v_add_u32_e32 v3, v9, v0
82 ; GFX9-NEXT: v_sub_u32_e32 v3, v3, v18
83 ; GFX9-NEXT: v_sub_u32_e32 v12, v12, v18
84 ; GFX9-NEXT: v_cmp_lt_u32_e64 s[4:5], v19, v13
85 ; GFX9-NEXT: v_mad_u64_u32 v[18:19], s[6:7], v19, v15, v[3:4]
86 ; GFX9-NEXT: v_cmp_lt_u32_e64 s[6:7], v12, v14
87 ; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
88 ; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], vcc
89 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, v18, s[4:5]
90 ; GFX9-NEXT: v_lshlrev_b64 v[18:19], 2, v[3:4]
91 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
92 ; GFX9-NEXT: v_add_co_u32_e64 v18, s[6:7], v10, v18
93 ; GFX9-NEXT: v_addc_co_u32_e64 v19, s[6:7], v11, v19, s[6:7]
94 ; GFX9-NEXT: global_load_dword v3, v[18:19], off
95 ; GFX9-NEXT: v_cmp_ge_u32_e64 s[6:7], v0, v1
96 ; GFX9-NEXT: s_or_b64 s[10:11], s[6:7], s[10:11]
97 ; GFX9-NEXT: s_waitcnt vmcnt(0)
98 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, v3, s[4:5]
99 ; GFX9-NEXT: ds_write_b32 v6, v3
100 ; GFX9-NEXT: v_add_u32_e32 v6, v6, v8
101 ; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
102 ; GFX9-NEXT: s_cbranch_execnz .LBB1_2
103 ; GFX9-NEXT: .LBB1_3: ; %Flow3
104 ; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
105 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
106 ; GFX9-NEXT: s_setpc_b64 s[30:31]
107 bb:
108 %tmp = icmp ult i32 %arg, %arg1
109 br i1 %tmp, label %bb19, label %.loopexit
111 bb19: ; preds = %bb
112 %tmp20 = uitofp i32 %arg6 to float
113 %tmp21 = fdiv float 1.000000e+00, %tmp20, !fpmath !0
114 %tmp22 = and i32 %arg6, 16777215
115 br label %bb23
117 .loopexit: ; preds = %bb23, %bb
118 ret void
120 bb23: ; preds = %bb19, %bb23
121 %tmp24 = phi i32 [ %arg, %bb19 ], [ %tmp47, %bb23 ]
122 %tmp25 = uitofp i32 %tmp24 to float
123 %tmp26 = tail call float @llvm.fmuladd.f32(float %tmp25, float %tmp21, float 0x3EE4F8B580000000) #2
124 %tmp27 = fptoui float %tmp26 to i32
125 %tmp28 = and i32 %tmp27, 16777215
126 %tmp29 = mul i32 %tmp28, %tmp22
127 %tmp30 = sub i32 %tmp24, %tmp29
128 %tmp31 = add i32 %tmp30, %arg16
129 %tmp32 = add i32 %tmp27, %arg15
130 %tmp33 = mul i32 %tmp32, %arg14
131 %tmp34 = add i32 %tmp33, %arg11
132 %tmp35 = add i32 %tmp34, %tmp31
133 %tmp36 = add i32 %tmp24, %arg4
134 %tmp37 = icmp ult i32 %tmp31, %arg13
135 %tmp38 = icmp ult i32 %tmp32, %arg12
136 %tmp39 = and i1 %tmp38, %tmp37
137 %tmp40 = and i1 %tmp39, %arg17
138 %tmp41 = zext i32 %tmp35 to i64
139 %tmp42 = select i1 %tmp40, i64 %tmp41, i64 0
140 %tmp43 = getelementptr inbounds float, float addrspace(1)* %arg10, i64 %tmp42
141 %tmp44 = load float, float addrspace(1)* %tmp43, align 4
142 %tmp45 = select i1 %tmp40, float %tmp44, float 0.000000e+00
143 %tmp46 = getelementptr inbounds float, float addrspace(3)* %arg3, i32 %tmp36
144 store float %tmp45, float addrspace(3)* %tmp46, align 4
145 %tmp47 = add i32 %tmp24, %arg2
146 %tmp48 = icmp ult i32 %tmp47, %arg1
147 br i1 %tmp48, label %bb23, label %.loopexit
148 }
150 define void @slsr1_0(i32 %b.arg, i32 %s.arg) #0 {
151 ; GFX9-LABEL: slsr1_0:
152 ; GFX9: ; %bb.0:
153 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
154 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff, v1
155 ; GFX9-NEXT: v_mul_u32_u24_e32 v3, v0, v1
156 ; GFX9-NEXT: global_store_dword v[0:1], v3, off
157 ; GFX9-NEXT: s_waitcnt vmcnt(0)
158 ; GFX9-NEXT: v_mad_u32_u24 v0, v0, v1, v2
159 ; GFX9-NEXT: global_store_dword v[0:1], v0, off
160 ; GFX9-NEXT: s_waitcnt vmcnt(0)
161 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
162 ; GFX9-NEXT: global_store_dword v[0:1], v0, off
163 ; GFX9-NEXT: s_waitcnt vmcnt(0)
164 ; GFX9-NEXT: s_setpc_b64 s[30:31]
165 %b = and i32 %b.arg, 16777215
166 %s = and i32 %s.arg, 16777215
168 ; CHECK-LABEL: @slsr1(
169 ; foo(b * s);
170 %mul0 = mul i32 %b, %s
171 ; CHECK: mul i32
172 ; CHECK-NOT: mul i32
173 store volatile i32 %mul0, i32 addrspace(1)* undef
175 ; foo((b + 1) * s);
176 %b1 = add i32 %b, 1
177 %mul1 = mul i32 %b1, %s
178 store volatile i32 %mul1, i32 addrspace(1)* undef
180 ; foo((b + 2) * s);
181 %b2 = add i32 %b, 2
182 %mul2 = mul i32 %b2, %s
183 store volatile i32 %mul2, i32 addrspace(1)* undef
184 ret void
185 }
187 define void @slsr1_1(i32 %b.arg, i32 %s.arg) #0 {
188 ; GFX9-LABEL: slsr1_1:
189 ; GFX9: ; %bb.0:
190 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
191 ; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
192 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
193 ; GFX9-NEXT: s_mov_b64 exec, s[4:5]
194 ; GFX9-NEXT: v_writelane_b32 v40, s33, 5
195 ; GFX9-NEXT: v_writelane_b32 v40, s30, 0
196 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1
197 ; GFX9-NEXT: s_mov_b32 s33, s32
198 ; GFX9-NEXT: s_addk_i32 s32, 0x800
199 ; GFX9-NEXT: v_writelane_b32 v40, s34, 2
200 ; GFX9-NEXT: v_writelane_b32 v40, s36, 3
201 ; GFX9-NEXT: s_getpc_b64 s[4:5]
202 ; GFX9-NEXT: s_add_u32 s4, s4, foo@gotpcrel32@lo+4
203 ; GFX9-NEXT: s_addc_u32 s5, s5, foo@gotpcrel32@hi+12
204 ; GFX9-NEXT: v_writelane_b32 v40, s37, 4
205 ; GFX9-NEXT: s_load_dwordx2 s[36:37], s[4:5], 0x0
206 ; GFX9-NEXT: buffer_store_dword v41, off, s[0:3], s33 offset:8 ; 4-byte Folded Spill
207 ; GFX9-NEXT: buffer_store_dword v42, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
208 ; GFX9-NEXT: buffer_store_dword v43, off, s[0:3], s33 ; 4-byte Folded Spill
209 ; GFX9-NEXT: v_mov_b32_e32 v41, v1
210 ; GFX9-NEXT: v_mov_b32_e32 v42, v0
211 ; GFX9-NEXT: v_mul_u32_u24_e32 v0, v42, v41
212 ; GFX9-NEXT: s_mov_b32 s34, s15
213 ; GFX9-NEXT: v_and_b32_e32 v43, 0xffffff, v41
214 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
215 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]
216 ; GFX9-NEXT: v_mad_u32_u24 v41, v42, v41, v43
217 ; GFX9-NEXT: s_mov_b32 s15, s34
218 ; GFX9-NEXT: v_mov_b32_e32 v0, v41
219 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]
220 ; GFX9-NEXT: v_add_u32_e32 v0, v41, v43
221 ; GFX9-NEXT: s_mov_b32 s15, s34
222 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37]
223 ; GFX9-NEXT: buffer_load_dword v43, off, s[0:3], s33 ; 4-byte Folded Reload
224 ; GFX9-NEXT: buffer_load_dword v42, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
225 ; GFX9-NEXT: buffer_load_dword v41, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
226 ; GFX9-NEXT: v_readlane_b32 s37, v40, 4
227 ; GFX9-NEXT: v_readlane_b32 s36, v40, 3
228 ; GFX9-NEXT: v_readlane_b32 s34, v40, 2
229 ; GFX9-NEXT: v_readlane_b32 s31, v40, 1
230 ; GFX9-NEXT: v_readlane_b32 s30, v40, 0
231 ; GFX9-NEXT: s_addk_i32 s32, 0xf800
232 ; GFX9-NEXT: v_readlane_b32 s33, v40, 5
233 ; GFX9-NEXT: s_or_saveexec_b64 s[4:5], -1
234 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
235 ; GFX9-NEXT: s_mov_b64 exec, s[4:5]
236 ; GFX9-NEXT: s_waitcnt vmcnt(0)
237 ; GFX9-NEXT: s_setpc_b64 s[30:31]
238 %b = and i32 %b.arg, 16777215
239 %s = and i32 %s.arg, 16777215
241 ; CHECK-LABEL: @slsr1(
242 ; foo(b * s);
243 %mul0 = mul i32 %b, %s
244 ; CHECK: mul i32
245 ; CHECK-NOT: mul i32
246 call void @foo(i32 %mul0)
248 ; foo((b + 1) * s);
249 %b1 = add i32 %b, 1
250 %mul1 = mul i32 %b1, %s
251 call void @foo(i32 %mul1)
253 ; foo((b + 2) * s);
254 %b2 = add i32 %b, 2
255 %mul2 = mul i32 %b2, %s
256 call void @foo(i32 %mul2)
258 ret void
259 }
261 declare void @foo(i32) #2
262 declare float @llvm.fmuladd.f32(float, float, float) #1
264 attributes #0 = { nounwind willreturn "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
265 attributes #1 = { nounwind readnone speculatable }
266 attributes #2 = { nounwind "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
268 !0 = !{float 2.500000e+00}