| blob | 415b2a10233fec5cd4dab6f2f6f9c0e1b8ae4d2c |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
2 ; RUN: llc -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=SI %s
3 ; RUN: llc -march=amdgcn -mcpu=fiji -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4 ; RUN: llc -march=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
5 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
6 ; RUN: llc -march=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
8 ; Make sure fdiv is promoted to f32.
10 define amdgpu_kernel void @v_fdiv_f16(
11 ; SI-LABEL: v_fdiv_f16:
12 ; SI: ; %bb.0: ; %entry
13 ; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
14 ; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
15 ; SI-NEXT: s_mov_b32 s3, 0xf000
16 ; SI-NEXT: s_mov_b32 s2, 0
17 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
18 ; SI-NEXT: s_waitcnt lgkmcnt(0)
19 ; SI-NEXT: s_mov_b64 s[0:1], s[6:7]
20 ; SI-NEXT: v_mov_b32_e32 v1, 0
21 ; SI-NEXT: s_mov_b64 s[10:11], s[2:3]
22 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[0:3], 0 addr64 glc
23 ; SI-NEXT: s_waitcnt vmcnt(0)
24 ; SI-NEXT: buffer_load_ushort v3, v[0:1], s[8:11], 0 addr64 glc
25 ; SI-NEXT: s_waitcnt vmcnt(0)
26 ; SI-NEXT: s_mov_b64 s[6:7], s[2:3]
27 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
28 ; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
29 ; SI-NEXT: v_div_scale_f32 v4, s[0:1], v3, v3, v2
30 ; SI-NEXT: v_rcp_f32_e32 v5, v4
31 ; SI-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
32 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
33 ; SI-NEXT: v_fma_f32 v7, -v4, v5, 1.0
34 ; SI-NEXT: v_fma_f32 v5, v7, v5, v5
35 ; SI-NEXT: v_mul_f32_e32 v7, v6, v5
36 ; SI-NEXT: v_fma_f32 v8, -v4, v7, v6
37 ; SI-NEXT: v_fma_f32 v7, v8, v5, v7
38 ; SI-NEXT: v_fma_f32 v4, -v4, v7, v6
39 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
40 ; SI-NEXT: v_div_fmas_f32 v4, v4, v5, v7
41 ; SI-NEXT: v_div_fixup_f32 v2, v4, v3, v2
42 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
43 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
44 ; SI-NEXT: buffer_store_short v2, v[0:1], s[4:7], 0 addr64
45 ; SI-NEXT: s_endpgm
46 ;
47 ; GFX8-LABEL: v_fdiv_f16:
48 ; GFX8: ; %bb.0: ; %entry
49 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
50 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
51 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 1, v0
52 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
53 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
54 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v4
55 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
56 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
57 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v4
58 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
59 ; GFX8-NEXT: flat_load_ushort v5, v[0:1] glc
60 ; GFX8-NEXT: s_waitcnt vmcnt(0)
61 ; GFX8-NEXT: flat_load_ushort v2, v[2:3] glc
62 ; GFX8-NEXT: s_waitcnt vmcnt(0)
63 ; GFX8-NEXT: v_mov_b32_e32 v3, s5
64 ; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v5
65 ; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v2
66 ; GFX8-NEXT: v_rcp_f32_e32 v0, v0
67 ; GFX8-NEXT: v_mul_f32_e32 v0, v1, v0
68 ; GFX8-NEXT: v_cvt_f16_f32_e32 v6, v0
69 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v4
70 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
71 ; GFX8-NEXT: v_div_fixup_f16 v2, v6, v2, v5
72 ; GFX8-NEXT: flat_store_short v[0:1], v2
73 ; GFX8-NEXT: s_endpgm
74 ;
75 ; GFX9-LABEL: v_fdiv_f16:
76 ; GFX9: ; %bb.0: ; %entry
77 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
78 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
79 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
80 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
81 ; GFX9-NEXT: global_load_ushort v1, v0, s[6:7] glc
82 ; GFX9-NEXT: s_waitcnt vmcnt(0)
83 ; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] glc
84 ; GFX9-NEXT: s_waitcnt vmcnt(0)
85 ; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v2
86 ; GFX9-NEXT: v_rcp_f32_e32 v3, v3
87 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
88 ; GFX9-NEXT: v_div_fixup_f16 v1, v3, v2, v1
89 ; GFX9-NEXT: global_store_short v0, v1, s[4:5]
90 ; GFX9-NEXT: s_endpgm
91 ;
92 ; GFX10-LABEL: v_fdiv_f16:
93 ; GFX10: ; %bb.0: ; %entry
94 ; GFX10-NEXT: s_clause 0x1
95 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
96 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
97 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
98 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
99 ; GFX10-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
100 ; GFX10-NEXT: s_waitcnt vmcnt(0)
101 ; GFX10-NEXT: global_load_ushort v2, v0, s[2:3] glc dlc
102 ; GFX10-NEXT: s_waitcnt vmcnt(0)
103 ; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2
104 ; GFX10-NEXT: v_rcp_f32_e32 v3, v3
105 ; GFX10-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
106 ; GFX10-NEXT: v_div_fixup_f16 v1, v3, v2, v1
107 ; GFX10-NEXT: global_store_short v0, v1, s[4:5]
108 ; GFX10-NEXT: s_endpgm
109 ;
110 ; GFX11-LABEL: v_fdiv_f16:
111 ; GFX11: ; %bb.0: ; %entry
112 ; GFX11-NEXT: s_clause 0x1
113 ; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
114 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
115 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
116 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
117 ; GFX11-NEXT: global_load_u16 v1, v0, s[6:7] glc dlc
118 ; GFX11-NEXT: s_waitcnt vmcnt(0)
119 ; GFX11-NEXT: global_load_u16 v2, v0, s[0:1] glc dlc
120 ; GFX11-NEXT: s_waitcnt vmcnt(0)
121 ; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v2
122 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
123 ; GFX11-NEXT: v_rcp_f32_e32 v3, v3
124 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
125 ; GFX11-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0]
126 ; GFX11-NEXT: v_div_fixup_f16 v1, v3, v2, v1
127 ; GFX11-NEXT: global_store_b16 v0, v1, s[4:5]
128 ; GFX11-NEXT: s_nop 0
129 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
130 ; GFX11-NEXT: s_endpgm
131 ptr addrspace(1) %r,
132 ptr addrspace(1) %a,
133 ptr addrspace(1) %b) #0 {
134 entry:
135 %tid = call i32 @llvm.amdgcn.workitem.id.x()
136 %tid.ext = sext i32 %tid to i64
137 %gep.a = getelementptr inbounds half, ptr addrspace(1) %a, i64 %tid.ext
138 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
139 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
140 %a.val = load volatile half, ptr addrspace(1) %gep.a
141 %b.val = load volatile half, ptr addrspace(1) %gep.b
142 %r.val = fdiv half %a.val, %b.val
143 store half %r.val, ptr addrspace(1) %gep.r
144 ret void
145 }
147 define amdgpu_kernel void @v_rcp_f16(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
148 ; SI-LABEL: v_rcp_f16:
149 ; SI: ; %bb.0: ; %entry
150 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
151 ; SI-NEXT: s_mov_b32 s7, 0xf000
152 ; SI-NEXT: s_mov_b32 s6, 0
153 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
154 ; SI-NEXT: v_mov_b32_e32 v1, 0
155 ; SI-NEXT: s_waitcnt lgkmcnt(0)
156 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
157 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
158 ; SI-NEXT: s_waitcnt vmcnt(0)
159 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
160 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
161 ; SI-NEXT: v_rcp_f32_e32 v4, v3
162 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
163 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
164 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
165 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
166 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
167 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
168 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
169 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
170 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
171 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
172 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
173 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
174 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
175 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
176 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
177 ; SI-NEXT: s_endpgm
178 ;
179 ; GFX8-LABEL: v_rcp_f16:
180 ; GFX8: ; %bb.0: ; %entry
181 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
182 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
183 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
184 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
185 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
186 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
187 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
188 ; GFX8-NEXT: s_waitcnt vmcnt(0)
189 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
190 ; GFX8-NEXT: v_rcp_f16_e32 v3, v0
191 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
192 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
193 ; GFX8-NEXT: flat_store_short v[0:1], v3
194 ; GFX8-NEXT: s_endpgm
195 ;
196 ; GFX9-LABEL: v_rcp_f16:
197 ; GFX9: ; %bb.0: ; %entry
198 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
199 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
200 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
201 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
202 ; GFX9-NEXT: s_waitcnt vmcnt(0)
203 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
204 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
205 ; GFX9-NEXT: s_endpgm
206 ;
207 ; GFX10-LABEL: v_rcp_f16:
208 ; GFX10: ; %bb.0: ; %entry
209 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
210 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
211 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
212 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
213 ; GFX10-NEXT: s_waitcnt vmcnt(0)
214 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
215 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
216 ; GFX10-NEXT: s_endpgm
217 ;
218 ; GFX11-LABEL: v_rcp_f16:
219 ; GFX11: ; %bb.0: ; %entry
220 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
221 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
222 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
223 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
224 ; GFX11-NEXT: s_waitcnt vmcnt(0)
225 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
226 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
227 ; GFX11-NEXT: s_nop 0
228 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
229 ; GFX11-NEXT: s_endpgm
230 entry:
231 %tid = call i32 @llvm.amdgcn.workitem.id.x()
232 %tid.ext = sext i32 %tid to i64
233 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
234 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
235 %b.val = load volatile half, ptr addrspace(1) %gep.b
236 %r.val = fdiv half 1.0, %b.val
237 store half %r.val, ptr addrspace(1) %gep.r
238 ret void
239 }
241 define amdgpu_kernel void @v_rcp_f16_abs(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
242 ; SI-LABEL: v_rcp_f16_abs:
243 ; SI: ; %bb.0: ; %entry
244 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
245 ; SI-NEXT: s_mov_b32 s7, 0xf000
246 ; SI-NEXT: s_mov_b32 s6, 0
247 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
248 ; SI-NEXT: v_mov_b32_e32 v1, 0
249 ; SI-NEXT: s_waitcnt lgkmcnt(0)
250 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
251 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
252 ; SI-NEXT: s_waitcnt vmcnt(0)
253 ; SI-NEXT: v_cvt_f32_f16_e64 v2, |v2|
254 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
255 ; SI-NEXT: v_rcp_f32_e32 v4, v3
256 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
257 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
258 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
259 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
260 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
261 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
262 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
263 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
264 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
265 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
266 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
267 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
268 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
269 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
270 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
271 ; SI-NEXT: s_endpgm
272 ;
273 ; GFX8-LABEL: v_rcp_f16_abs:
274 ; GFX8: ; %bb.0: ; %entry
275 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
276 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
277 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
278 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
279 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
280 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
281 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
282 ; GFX8-NEXT: s_waitcnt vmcnt(0)
283 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
284 ; GFX8-NEXT: v_rcp_f16_e64 v3, |v0|
285 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
286 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
287 ; GFX8-NEXT: flat_store_short v[0:1], v3
288 ; GFX8-NEXT: s_endpgm
289 ;
290 ; GFX9-LABEL: v_rcp_f16_abs:
291 ; GFX9: ; %bb.0: ; %entry
292 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
293 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
294 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
295 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
296 ; GFX9-NEXT: s_waitcnt vmcnt(0)
297 ; GFX9-NEXT: v_rcp_f16_e64 v1, |v1|
298 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
299 ; GFX9-NEXT: s_endpgm
300 ;
301 ; GFX10-LABEL: v_rcp_f16_abs:
302 ; GFX10: ; %bb.0: ; %entry
303 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
304 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
305 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
306 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
307 ; GFX10-NEXT: s_waitcnt vmcnt(0)
308 ; GFX10-NEXT: v_rcp_f16_e64 v1, |v1|
309 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
310 ; GFX10-NEXT: s_endpgm
311 ;
312 ; GFX11-LABEL: v_rcp_f16_abs:
313 ; GFX11: ; %bb.0: ; %entry
314 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
315 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
316 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
317 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
318 ; GFX11-NEXT: s_waitcnt vmcnt(0)
319 ; GFX11-NEXT: v_rcp_f16_e64 v1, |v1|
320 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
321 ; GFX11-NEXT: s_nop 0
322 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
323 ; GFX11-NEXT: s_endpgm
324 entry:
325 %tid = call i32 @llvm.amdgcn.workitem.id.x()
326 %tid.ext = sext i32 %tid to i64
327 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
328 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
329 %b.val = load volatile half, ptr addrspace(1) %gep.b
330 %b.abs = call half @llvm.fabs.f16(half %b.val)
331 %r.val = fdiv half 1.0, %b.abs
332 store half %r.val, ptr addrspace(1) %gep.r
333 ret void
334 }
336 ; We could not do 1/b -> rcp_f32(b) under !fpmath < 1ulp.
338 define amdgpu_kernel void @reciprocal_f16_rounded(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
339 ; SI-LABEL: reciprocal_f16_rounded:
340 ; SI: ; %bb.0: ; %entry
341 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
342 ; SI-NEXT: s_mov_b32 s7, 0xf000
343 ; SI-NEXT: s_mov_b32 s6, 0
344 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
345 ; SI-NEXT: v_mov_b32_e32 v1, 0
346 ; SI-NEXT: s_waitcnt lgkmcnt(0)
347 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
348 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
349 ; SI-NEXT: s_waitcnt vmcnt(0)
350 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
351 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
352 ; SI-NEXT: v_rcp_f32_e32 v4, v3
353 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
354 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
355 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
356 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
357 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
358 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
359 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
360 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
361 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
362 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
363 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
364 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
365 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
366 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
367 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
368 ; SI-NEXT: s_endpgm
369 ;
370 ; GFX8-LABEL: reciprocal_f16_rounded:
371 ; GFX8: ; %bb.0: ; %entry
372 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
373 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
374 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
375 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
376 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
377 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
378 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
379 ; GFX8-NEXT: s_waitcnt vmcnt(0)
380 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
381 ; GFX8-NEXT: v_rcp_f16_e32 v3, v0
382 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
383 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
384 ; GFX8-NEXT: flat_store_short v[0:1], v3
385 ; GFX8-NEXT: s_endpgm
386 ;
387 ; GFX9-LABEL: reciprocal_f16_rounded:
388 ; GFX9: ; %bb.0: ; %entry
389 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
390 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
391 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
392 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
393 ; GFX9-NEXT: s_waitcnt vmcnt(0)
394 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
395 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
396 ; GFX9-NEXT: s_endpgm
397 ;
398 ; GFX10-LABEL: reciprocal_f16_rounded:
399 ; GFX10: ; %bb.0: ; %entry
400 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
401 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
402 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
403 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
404 ; GFX10-NEXT: s_waitcnt vmcnt(0)
405 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
406 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
407 ; GFX10-NEXT: s_endpgm
408 ;
409 ; GFX11-LABEL: reciprocal_f16_rounded:
410 ; GFX11: ; %bb.0: ; %entry
411 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
412 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
413 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
414 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
415 ; GFX11-NEXT: s_waitcnt vmcnt(0)
416 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
417 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
418 ; GFX11-NEXT: s_nop 0
419 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
420 ; GFX11-NEXT: s_endpgm
421 entry:
422 %tid = call i32 @llvm.amdgcn.workitem.id.x()
423 %tid.ext = sext i32 %tid to i64
424 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
425 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
426 %b.val = load volatile half, ptr addrspace(1) %gep.b
427 %r.val = fdiv half 1.0, %b.val
428 store half %r.val, ptr addrspace(1) %gep.r
429 ret void
430 }
432 define amdgpu_kernel void @v_rcp_f16_afn(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
433 ; SI-LABEL: v_rcp_f16_afn:
434 ; SI: ; %bb.0: ; %entry
435 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
436 ; SI-NEXT: s_mov_b32 s7, 0xf000
437 ; SI-NEXT: s_mov_b32 s6, 0
438 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
439 ; SI-NEXT: v_mov_b32_e32 v1, 0
440 ; SI-NEXT: s_waitcnt lgkmcnt(0)
441 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
442 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
443 ; SI-NEXT: s_waitcnt vmcnt(0)
444 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
445 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
446 ; SI-NEXT: v_rcp_f32_e32 v2, v2
447 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
448 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
449 ; SI-NEXT: s_endpgm
450 ;
451 ; GFX8-LABEL: v_rcp_f16_afn:
452 ; GFX8: ; %bb.0: ; %entry
453 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
454 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
455 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
456 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
457 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
458 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
459 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
460 ; GFX8-NEXT: s_waitcnt vmcnt(0)
461 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
462 ; GFX8-NEXT: v_rcp_f16_e32 v3, v0
463 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
464 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
465 ; GFX8-NEXT: flat_store_short v[0:1], v3
466 ; GFX8-NEXT: s_endpgm
467 ;
468 ; GFX9-LABEL: v_rcp_f16_afn:
469 ; GFX9: ; %bb.0: ; %entry
470 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
471 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
472 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
473 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
474 ; GFX9-NEXT: s_waitcnt vmcnt(0)
475 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
476 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
477 ; GFX9-NEXT: s_endpgm
478 ;
479 ; GFX10-LABEL: v_rcp_f16_afn:
480 ; GFX10: ; %bb.0: ; %entry
481 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
482 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
483 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
484 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
485 ; GFX10-NEXT: s_waitcnt vmcnt(0)
486 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
487 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
488 ; GFX10-NEXT: s_endpgm
489 ;
490 ; GFX11-LABEL: v_rcp_f16_afn:
491 ; GFX11: ; %bb.0: ; %entry
492 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
493 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
494 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
495 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
496 ; GFX11-NEXT: s_waitcnt vmcnt(0)
497 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
498 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
499 ; GFX11-NEXT: s_nop 0
500 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
501 ; GFX11-NEXT: s_endpgm
502 entry:
503 %tid = call i32 @llvm.amdgcn.workitem.id.x()
504 %tid.ext = sext i32 %tid to i64
505 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
506 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
507 %b.val = load volatile half, ptr addrspace(1) %gep.b
508 %r.val = fdiv afn half 1.0, %b.val
509 store half %r.val, ptr addrspace(1) %gep.r
510 ret void
511 }
513 define amdgpu_kernel void @v_rcp_f16_neg(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
514 ; SI-LABEL: v_rcp_f16_neg:
515 ; SI: ; %bb.0: ; %entry
516 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
517 ; SI-NEXT: s_mov_b32 s7, 0xf000
518 ; SI-NEXT: s_mov_b32 s6, 0
519 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
520 ; SI-NEXT: v_mov_b32_e32 v1, 0
521 ; SI-NEXT: s_waitcnt lgkmcnt(0)
522 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
523 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
524 ; SI-NEXT: s_waitcnt vmcnt(0)
525 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
526 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, -1.0
527 ; SI-NEXT: v_rcp_f32_e32 v4, v3
528 ; SI-NEXT: v_div_scale_f32 v5, vcc, -1.0, v2, -1.0
529 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
530 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
531 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
532 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
533 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
534 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
535 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
536 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
537 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
538 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, -1.0
539 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
540 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
541 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
542 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
543 ; SI-NEXT: s_endpgm
544 ;
545 ; GFX8-LABEL: v_rcp_f16_neg:
546 ; GFX8: ; %bb.0: ; %entry
547 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
548 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
549 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
550 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
551 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
552 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
553 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
554 ; GFX8-NEXT: s_waitcnt vmcnt(0)
555 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
556 ; GFX8-NEXT: v_rcp_f16_e64 v3, -v0
557 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
558 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
559 ; GFX8-NEXT: flat_store_short v[0:1], v3
560 ; GFX8-NEXT: s_endpgm
561 ;
562 ; GFX9-LABEL: v_rcp_f16_neg:
563 ; GFX9: ; %bb.0: ; %entry
564 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
565 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
566 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
567 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
568 ; GFX9-NEXT: s_waitcnt vmcnt(0)
569 ; GFX9-NEXT: v_rcp_f16_e64 v1, -v1
570 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
571 ; GFX9-NEXT: s_endpgm
572 ;
573 ; GFX10-LABEL: v_rcp_f16_neg:
574 ; GFX10: ; %bb.0: ; %entry
575 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
576 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
577 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
578 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
579 ; GFX10-NEXT: s_waitcnt vmcnt(0)
580 ; GFX10-NEXT: v_rcp_f16_e64 v1, -v1
581 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
582 ; GFX10-NEXT: s_endpgm
583 ;
584 ; GFX11-LABEL: v_rcp_f16_neg:
585 ; GFX11: ; %bb.0: ; %entry
586 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
587 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
588 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
589 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
590 ; GFX11-NEXT: s_waitcnt vmcnt(0)
591 ; GFX11-NEXT: v_rcp_f16_e64 v1, -v1
592 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
593 ; GFX11-NEXT: s_nop 0
594 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
595 ; GFX11-NEXT: s_endpgm
596 entry:
597 %tid = call i32 @llvm.amdgcn.workitem.id.x()
598 %tid.ext = sext i32 %tid to i64
599 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
600 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
601 %b.val = load volatile half, ptr addrspace(1) %gep.b
602 %r.val = fdiv half -1.0, %b.val
603 store half %r.val, ptr addrspace(1) %gep.r
604 ret void
605 }
607 define amdgpu_kernel void @v_rsq_f16(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
608 ; SI-LABEL: v_rsq_f16:
609 ; SI: ; %bb.0: ; %entry
610 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
611 ; SI-NEXT: s_mov_b32 s7, 0xf000
612 ; SI-NEXT: s_mov_b32 s6, 0
613 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
614 ; SI-NEXT: v_mov_b32_e32 v1, 0
615 ; SI-NEXT: s_waitcnt lgkmcnt(0)
616 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
617 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
618 ; SI-NEXT: s_waitcnt vmcnt(0)
619 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
620 ; SI-NEXT: v_sqrt_f32_e32 v2, v2
621 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
622 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
623 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
624 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
625 ; SI-NEXT: v_rcp_f32_e32 v4, v3
626 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
627 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
628 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
629 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
630 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
631 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
632 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
633 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
634 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
635 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
636 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
637 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
638 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
639 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
640 ; SI-NEXT: s_endpgm
641 ;
642 ; GFX8-LABEL: v_rsq_f16:
643 ; GFX8: ; %bb.0: ; %entry
644 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
645 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
646 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
647 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
648 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
649 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
650 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
651 ; GFX8-NEXT: s_waitcnt vmcnt(0)
652 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
653 ; GFX8-NEXT: v_rsq_f16_e32 v3, v0
654 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
655 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
656 ; GFX8-NEXT: flat_store_short v[0:1], v3
657 ; GFX8-NEXT: s_endpgm
658 ;
659 ; GFX9-LABEL: v_rsq_f16:
660 ; GFX9: ; %bb.0: ; %entry
661 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
662 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
663 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
664 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
665 ; GFX9-NEXT: s_waitcnt vmcnt(0)
666 ; GFX9-NEXT: v_rsq_f16_e32 v1, v1
667 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
668 ; GFX9-NEXT: s_endpgm
669 ;
670 ; GFX10-LABEL: v_rsq_f16:
671 ; GFX10: ; %bb.0: ; %entry
672 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
673 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
674 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
675 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
676 ; GFX10-NEXT: s_waitcnt vmcnt(0)
677 ; GFX10-NEXT: v_rsq_f16_e32 v1, v1
678 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
679 ; GFX10-NEXT: s_endpgm
680 ;
681 ; GFX11-LABEL: v_rsq_f16:
682 ; GFX11: ; %bb.0: ; %entry
683 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
684 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
685 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
686 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
687 ; GFX11-NEXT: s_waitcnt vmcnt(0)
688 ; GFX11-NEXT: v_rsq_f16_e32 v1, v1
689 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
690 ; GFX11-NEXT: s_nop 0
691 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
692 ; GFX11-NEXT: s_endpgm
693 entry:
694 %tid = call i32 @llvm.amdgcn.workitem.id.x()
695 %tid.ext = sext i32 %tid to i64
696 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
697 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
698 %b.val = load volatile half, ptr addrspace(1) %gep.b
699 %b.sqrt = call contract half @llvm.sqrt.f16(half %b.val)
700 %r.val = fdiv contract half 1.0, %b.sqrt
701 store half %r.val, ptr addrspace(1) %gep.r
702 ret void
703 }
705 define amdgpu_kernel void @v_rsq_f16_neg(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
706 ; SI-LABEL: v_rsq_f16_neg:
707 ; SI: ; %bb.0: ; %entry
708 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
709 ; SI-NEXT: s_mov_b32 s7, 0xf000
710 ; SI-NEXT: s_mov_b32 s6, 0
711 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
712 ; SI-NEXT: v_mov_b32_e32 v1, 0
713 ; SI-NEXT: s_waitcnt lgkmcnt(0)
714 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
715 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
716 ; SI-NEXT: s_waitcnt vmcnt(0)
717 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
718 ; SI-NEXT: v_sqrt_f32_e32 v2, v2
719 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
720 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
721 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
722 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, -1.0
723 ; SI-NEXT: v_rcp_f32_e32 v4, v3
724 ; SI-NEXT: v_div_scale_f32 v5, vcc, -1.0, v2, -1.0
725 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
726 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
727 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
728 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
729 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
730 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
731 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
732 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
733 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
734 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, -1.0
735 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
736 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
737 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
738 ; SI-NEXT: s_endpgm
739 ;
740 ; GFX8-LABEL: v_rsq_f16_neg:
741 ; GFX8: ; %bb.0: ; %entry
742 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
743 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
744 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
745 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
746 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
747 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
748 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
749 ; GFX8-NEXT: s_waitcnt vmcnt(0)
750 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
751 ; GFX8-NEXT: v_rsq_f16_e32 v3, v0
752 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
753 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
754 ; GFX8-NEXT: v_xor_b32_e32 v2, 0x8000, v3
755 ; GFX8-NEXT: flat_store_short v[0:1], v2
756 ; GFX8-NEXT: s_endpgm
757 ;
758 ; GFX9-LABEL: v_rsq_f16_neg:
759 ; GFX9: ; %bb.0: ; %entry
760 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
761 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
762 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
763 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
764 ; GFX9-NEXT: s_waitcnt vmcnt(0)
765 ; GFX9-NEXT: v_rsq_f16_e32 v1, v1
766 ; GFX9-NEXT: v_xor_b32_e32 v1, 0x8000, v1
767 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
768 ; GFX9-NEXT: s_endpgm
769 ;
770 ; GFX10-LABEL: v_rsq_f16_neg:
771 ; GFX10: ; %bb.0: ; %entry
772 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
773 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
774 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
775 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
776 ; GFX10-NEXT: s_waitcnt vmcnt(0)
777 ; GFX10-NEXT: v_rsq_f16_e32 v1, v1
778 ; GFX10-NEXT: v_xor_b32_e32 v1, 0x8000, v1
779 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
780 ; GFX10-NEXT: s_endpgm
781 ;
782 ; GFX11-LABEL: v_rsq_f16_neg:
783 ; GFX11: ; %bb.0: ; %entry
784 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
785 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
786 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
787 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
788 ; GFX11-NEXT: s_waitcnt vmcnt(0)
789 ; GFX11-NEXT: v_rsq_f16_e32 v1, v1
790 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
791 ; GFX11-NEXT: v_xor_b32_e32 v1, 0x8000, v1
792 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
793 ; GFX11-NEXT: s_nop 0
794 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
795 ; GFX11-NEXT: s_endpgm
796 entry:
797 %tid = call i32 @llvm.amdgcn.workitem.id.x()
798 %tid.ext = sext i32 %tid to i64
799 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
800 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
801 %b.val = load volatile half, ptr addrspace(1) %gep.b
802 %b.sqrt = call contract half @llvm.sqrt.f16(half %b.val)
803 %r.val = fdiv contract half -1.0, %b.sqrt
804 store half %r.val, ptr addrspace(1) %gep.r
805 ret void
806 }
808 define amdgpu_kernel void @v_rsq_f16_multi_use(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
809 ; SI-LABEL: v_rsq_f16_multi_use:
810 ; SI: ; %bb.0: ; %entry
811 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
812 ; SI-NEXT: s_mov_b32 s7, 0xf000
813 ; SI-NEXT: s_mov_b32 s6, 0
814 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
815 ; SI-NEXT: v_mov_b32_e32 v1, 0
816 ; SI-NEXT: s_waitcnt lgkmcnt(0)
817 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
818 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
819 ; SI-NEXT: s_waitcnt vmcnt(0)
820 ; SI-NEXT: v_cvt_f32_f16_e32 v3, v2
821 ; SI-NEXT: v_sqrt_f32_e32 v3, v3
822 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
823 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
824 ; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
825 ; SI-NEXT: v_div_scale_f32 v4, s[2:3], v3, v3, 1.0
826 ; SI-NEXT: v_rcp_f32_e32 v5, v4
827 ; SI-NEXT: v_div_scale_f32 v6, vcc, 1.0, v3, 1.0
828 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
829 ; SI-NEXT: v_fma_f32 v7, -v4, v5, 1.0
830 ; SI-NEXT: v_fma_f32 v5, v7, v5, v5
831 ; SI-NEXT: v_mul_f32_e32 v7, v6, v5
832 ; SI-NEXT: v_fma_f32 v8, -v4, v7, v6
833 ; SI-NEXT: v_fma_f32 v7, v8, v5, v7
834 ; SI-NEXT: v_fma_f32 v4, -v4, v7, v6
835 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
836 ; SI-NEXT: v_div_fmas_f32 v4, v4, v5, v7
837 ; SI-NEXT: v_div_fixup_f32 v3, v4, v3, 1.0
838 ; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
839 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
840 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
841 ; SI-NEXT: s_waitcnt vmcnt(0)
842 ; SI-NEXT: buffer_store_short v3, v[0:1], s[0:3], 0 addr64
843 ; SI-NEXT: s_endpgm
844 ;
845 ; GFX8-LABEL: v_rsq_f16_multi_use:
846 ; GFX8: ; %bb.0: ; %entry
847 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
848 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
849 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
850 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
851 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
852 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
853 ; GFX8-NEXT: flat_load_ushort v3, v[0:1] glc
854 ; GFX8-NEXT: s_waitcnt vmcnt(0)
855 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
856 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
857 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
858 ; GFX8-NEXT: v_rsq_f16_e32 v4, v3
859 ; GFX8-NEXT: flat_store_short v[0:1], v3
860 ; GFX8-NEXT: s_waitcnt vmcnt(0)
861 ; GFX8-NEXT: flat_store_short v[0:1], v4
862 ; GFX8-NEXT: s_endpgm
863 ;
864 ; GFX9-LABEL: v_rsq_f16_multi_use:
865 ; GFX9: ; %bb.0: ; %entry
866 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
867 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
868 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
869 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
870 ; GFX9-NEXT: s_waitcnt vmcnt(0)
871 ; GFX9-NEXT: v_rsq_f16_e32 v2, v1
872 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
873 ; GFX9-NEXT: s_waitcnt vmcnt(0)
874 ; GFX9-NEXT: global_store_short v0, v2, s[0:1]
875 ; GFX9-NEXT: s_endpgm
876 ;
877 ; GFX10-LABEL: v_rsq_f16_multi_use:
878 ; GFX10: ; %bb.0: ; %entry
879 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
880 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
881 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
882 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
883 ; GFX10-NEXT: s_waitcnt vmcnt(0)
884 ; GFX10-NEXT: v_rsq_f16_e32 v2, v1
885 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
886 ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
887 ; GFX10-NEXT: global_store_short v0, v2, s[0:1]
888 ; GFX10-NEXT: s_endpgm
889 ;
890 ; GFX11-LABEL: v_rsq_f16_multi_use:
891 ; GFX11: ; %bb.0: ; %entry
892 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
893 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
894 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
895 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
896 ; GFX11-NEXT: s_waitcnt vmcnt(0)
897 ; GFX11-NEXT: v_rsq_f16_e32 v2, v1
898 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1] dlc
899 ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
900 ; GFX11-NEXT: global_store_b16 v0, v2, s[0:1]
901 ; GFX11-NEXT: s_nop 0
902 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
903 ; GFX11-NEXT: s_endpgm
904 entry:
905 %tid = call i32 @llvm.amdgcn.workitem.id.x()
906 %tid.ext = sext i32 %tid to i64
907 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
908 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
909 %b.val = load volatile half, ptr addrspace(1) %gep.b
910 store volatile half %b.val, ptr addrspace(1) %gep.r
911 %b.sqrt = call contract half @llvm.sqrt.f16(half %b.val)
912 %r.val = fdiv contract half 1.0, %b.sqrt
913 store half %r.val, ptr addrspace(1) %gep.r
914 ret void
915 }
917 define amdgpu_kernel void @v_rsq_f16_missing_contract0(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
918 ; SI-LABEL: v_rsq_f16_missing_contract0:
919 ; SI: ; %bb.0: ; %entry
920 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
921 ; SI-NEXT: s_mov_b32 s7, 0xf000
922 ; SI-NEXT: s_mov_b32 s6, 0
923 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
924 ; SI-NEXT: v_mov_b32_e32 v1, 0
925 ; SI-NEXT: s_waitcnt lgkmcnt(0)
926 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
927 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
928 ; SI-NEXT: s_waitcnt vmcnt(0)
929 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
930 ; SI-NEXT: v_sqrt_f32_e32 v2, v2
931 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
932 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
933 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
934 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
935 ; SI-NEXT: v_rcp_f32_e32 v4, v3
936 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
937 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
938 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
939 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
940 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
941 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
942 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
943 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
944 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
945 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
946 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
947 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
948 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
949 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
950 ; SI-NEXT: s_endpgm
951 ;
952 ; GFX8-LABEL: v_rsq_f16_missing_contract0:
953 ; GFX8: ; %bb.0: ; %entry
954 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
955 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
956 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
957 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
958 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
959 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
960 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
961 ; GFX8-NEXT: s_waitcnt vmcnt(0)
962 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
963 ; GFX8-NEXT: v_sqrt_f16_e32 v0, v0
964 ; GFX8-NEXT: v_rcp_f16_e32 v3, v0
965 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
966 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
967 ; GFX8-NEXT: flat_store_short v[0:1], v3
968 ; GFX8-NEXT: s_endpgm
969 ;
970 ; GFX9-LABEL: v_rsq_f16_missing_contract0:
971 ; GFX9: ; %bb.0: ; %entry
972 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
973 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
974 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
975 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
976 ; GFX9-NEXT: s_waitcnt vmcnt(0)
977 ; GFX9-NEXT: v_sqrt_f16_e32 v1, v1
978 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
979 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
980 ; GFX9-NEXT: s_endpgm
981 ;
982 ; GFX10-LABEL: v_rsq_f16_missing_contract0:
983 ; GFX10: ; %bb.0: ; %entry
984 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
985 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
986 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
987 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
988 ; GFX10-NEXT: s_waitcnt vmcnt(0)
989 ; GFX10-NEXT: v_sqrt_f16_e32 v1, v1
990 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
991 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
992 ; GFX10-NEXT: s_endpgm
993 ;
994 ; GFX11-LABEL: v_rsq_f16_missing_contract0:
995 ; GFX11: ; %bb.0: ; %entry
996 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
997 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
998 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
999 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
1000 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1001 ; GFX11-NEXT: v_sqrt_f16_e32 v1, v1
1002 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1003 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
1004 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
1005 ; GFX11-NEXT: s_nop 0
1006 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1007 ; GFX11-NEXT: s_endpgm
1008 entry:
1009 %tid = call i32 @llvm.amdgcn.workitem.id.x()
1010 %tid.ext = sext i32 %tid to i64
1011 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
1012 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
1013 %b.val = load volatile half, ptr addrspace(1) %gep.b
1014 %b.sqrt = call half @llvm.sqrt.f16(half %b.val)
1015 %r.val = fdiv contract half 1.0, %b.sqrt
1016 store half %r.val, ptr addrspace(1) %gep.r
1017 ret void
1018 }
1020 define amdgpu_kernel void @v_rsq_f16_missing_contract1(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
1021 ; SI-LABEL: v_rsq_f16_missing_contract1:
1022 ; SI: ; %bb.0: ; %entry
1023 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
1024 ; SI-NEXT: s_mov_b32 s7, 0xf000
1025 ; SI-NEXT: s_mov_b32 s6, 0
1026 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1027 ; SI-NEXT: v_mov_b32_e32 v1, 0
1028 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1029 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
1030 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
1031 ; SI-NEXT: s_waitcnt vmcnt(0)
1032 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1033 ; SI-NEXT: v_sqrt_f32_e32 v2, v2
1034 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1035 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1036 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1037 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, 1.0
1038 ; SI-NEXT: v_rcp_f32_e32 v4, v3
1039 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v2, 1.0
1040 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1041 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
1042 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
1043 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
1044 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
1045 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
1046 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
1047 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1048 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
1049 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, 1.0
1050 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1051 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
1052 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
1053 ; SI-NEXT: s_endpgm
1054 ;
1055 ; GFX8-LABEL: v_rsq_f16_missing_contract1:
1056 ; GFX8: ; %bb.0: ; %entry
1057 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1058 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
1059 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1060 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
1061 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
1062 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1063 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
1064 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1065 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1066 ; GFX8-NEXT: v_sqrt_f16_e32 v0, v0
1067 ; GFX8-NEXT: v_rcp_f16_e32 v3, v0
1068 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
1069 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1070 ; GFX8-NEXT: flat_store_short v[0:1], v3
1071 ; GFX8-NEXT: s_endpgm
1072 ;
1073 ; GFX9-LABEL: v_rsq_f16_missing_contract1:
1074 ; GFX9: ; %bb.0: ; %entry
1075 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1076 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1077 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1078 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
1079 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1080 ; GFX9-NEXT: v_sqrt_f16_e32 v1, v1
1081 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
1082 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
1083 ; GFX9-NEXT: s_endpgm
1084 ;
1085 ; GFX10-LABEL: v_rsq_f16_missing_contract1:
1086 ; GFX10: ; %bb.0: ; %entry
1087 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1088 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1089 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1090 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
1091 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1092 ; GFX10-NEXT: v_sqrt_f16_e32 v1, v1
1093 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
1094 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
1095 ; GFX10-NEXT: s_endpgm
1096 ;
1097 ; GFX11-LABEL: v_rsq_f16_missing_contract1:
1098 ; GFX11: ; %bb.0: ; %entry
1099 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
1100 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1101 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1102 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
1103 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1104 ; GFX11-NEXT: v_sqrt_f16_e32 v1, v1
1105 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1106 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
1107 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
1108 ; GFX11-NEXT: s_nop 0
1109 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1110 ; GFX11-NEXT: s_endpgm
1111 entry:
1112 %tid = call i32 @llvm.amdgcn.workitem.id.x()
1113 %tid.ext = sext i32 %tid to i64
1114 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
1115 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
1116 %b.val = load volatile half, ptr addrspace(1) %gep.b
1117 %b.sqrt = call contract half @llvm.sqrt.f16(half %b.val)
1118 %r.val = fdiv half 1.0, %b.sqrt
1119 store half %r.val, ptr addrspace(1) %gep.r
1120 ret void
1121 }
1123 define amdgpu_kernel void @v_neg_rsq_f16_missing_contract1(ptr addrspace(1) %r, ptr addrspace(1) %b) #0 {
1124 ; SI-LABEL: v_neg_rsq_f16_missing_contract1:
1125 ; SI: ; %bb.0: ; %entry
1126 ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
1127 ; SI-NEXT: s_mov_b32 s7, 0xf000
1128 ; SI-NEXT: s_mov_b32 s6, 0
1129 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1130 ; SI-NEXT: v_mov_b32_e32 v1, 0
1131 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1132 ; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
1133 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[4:7], 0 addr64 glc
1134 ; SI-NEXT: s_waitcnt vmcnt(0)
1135 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1136 ; SI-NEXT: v_sqrt_f32_e32 v2, v2
1137 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1138 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1139 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1140 ; SI-NEXT: v_div_scale_f32 v3, s[2:3], v2, v2, -1.0
1141 ; SI-NEXT: v_rcp_f32_e32 v4, v3
1142 ; SI-NEXT: v_div_scale_f32 v5, vcc, -1.0, v2, -1.0
1143 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1144 ; SI-NEXT: v_fma_f32 v6, -v3, v4, 1.0
1145 ; SI-NEXT: v_fma_f32 v4, v6, v4, v4
1146 ; SI-NEXT: v_mul_f32_e32 v6, v5, v4
1147 ; SI-NEXT: v_fma_f32 v7, -v3, v6, v5
1148 ; SI-NEXT: v_fma_f32 v6, v7, v4, v6
1149 ; SI-NEXT: v_fma_f32 v3, -v3, v6, v5
1150 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1151 ; SI-NEXT: v_div_fmas_f32 v3, v3, v4, v6
1152 ; SI-NEXT: v_div_fixup_f32 v2, v3, v2, -1.0
1153 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1154 ; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
1155 ; SI-NEXT: buffer_store_short v2, v[0:1], s[0:3], 0 addr64
1156 ; SI-NEXT: s_endpgm
1157 ;
1158 ; GFX8-LABEL: v_neg_rsq_f16_missing_contract1:
1159 ; GFX8: ; %bb.0: ; %entry
1160 ; GFX8-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1161 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 1, v0
1162 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1163 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
1164 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
1165 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1166 ; GFX8-NEXT: flat_load_ushort v0, v[0:1] glc
1167 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1168 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1169 ; GFX8-NEXT: v_sqrt_f16_e32 v0, v0
1170 ; GFX8-NEXT: v_rcp_f16_e64 v3, -v0
1171 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
1172 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1173 ; GFX8-NEXT: flat_store_short v[0:1], v3
1174 ; GFX8-NEXT: s_endpgm
1175 ;
1176 ; GFX9-LABEL: v_neg_rsq_f16_missing_contract1:
1177 ; GFX9: ; %bb.0: ; %entry
1178 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1179 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1180 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1181 ; GFX9-NEXT: global_load_ushort v1, v0, s[2:3] glc
1182 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1183 ; GFX9-NEXT: v_sqrt_f16_e32 v1, v1
1184 ; GFX9-NEXT: v_rcp_f16_e64 v1, -v1
1185 ; GFX9-NEXT: global_store_short v0, v1, s[0:1]
1186 ; GFX9-NEXT: s_endpgm
1187 ;
1188 ; GFX10-LABEL: v_neg_rsq_f16_missing_contract1:
1189 ; GFX10: ; %bb.0: ; %entry
1190 ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
1191 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1192 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1193 ; GFX10-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
1194 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1195 ; GFX10-NEXT: v_sqrt_f16_e32 v1, v1
1196 ; GFX10-NEXT: v_rcp_f16_e64 v1, -v1
1197 ; GFX10-NEXT: global_store_short v0, v1, s[0:1]
1198 ; GFX10-NEXT: s_endpgm
1199 ;
1200 ; GFX11-LABEL: v_neg_rsq_f16_missing_contract1:
1201 ; GFX11: ; %bb.0: ; %entry
1202 ; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x24
1203 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1204 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1205 ; GFX11-NEXT: global_load_u16 v1, v0, s[2:3] glc dlc
1206 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1207 ; GFX11-NEXT: v_sqrt_f16_e32 v1, v1
1208 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1209 ; GFX11-NEXT: v_rcp_f16_e64 v1, -v1
1210 ; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
1211 ; GFX11-NEXT: s_nop 0
1212 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1213 ; GFX11-NEXT: s_endpgm
1214 entry:
1215 %tid = call i32 @llvm.amdgcn.workitem.id.x()
1216 %tid.ext = sext i32 %tid to i64
1217 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
1218 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
1219 %b.val = load volatile half, ptr addrspace(1) %gep.b
1220 %b.sqrt = call contract half @llvm.sqrt.f16(half %b.val)
1221 %r.val = fdiv half -1.0, %b.sqrt
1222 store half %r.val, ptr addrspace(1) %gep.r
1223 ret void
1224 }
1226 define amdgpu_kernel void @v_fdiv_f16_afn(ptr addrspace(1) %r, ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {
1227 ; SI-LABEL: v_fdiv_f16_afn:
1228 ; SI: ; %bb.0: ; %entry
1229 ; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1230 ; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
1231 ; SI-NEXT: s_mov_b32 s3, 0xf000
1232 ; SI-NEXT: s_mov_b32 s2, 0
1233 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1234 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1235 ; SI-NEXT: s_mov_b64 s[0:1], s[6:7]
1236 ; SI-NEXT: v_mov_b32_e32 v1, 0
1237 ; SI-NEXT: s_mov_b64 s[10:11], s[2:3]
1238 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[0:3], 0 addr64 glc
1239 ; SI-NEXT: s_waitcnt vmcnt(0)
1240 ; SI-NEXT: buffer_load_ushort v3, v[0:1], s[8:11], 0 addr64 glc
1241 ; SI-NEXT: s_waitcnt vmcnt(0)
1242 ; SI-NEXT: s_mov_b64 s[6:7], s[2:3]
1243 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1244 ; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
1245 ; SI-NEXT: v_rcp_f32_e32 v3, v3
1246 ; SI-NEXT: v_mul_f32_e32 v2, v2, v3
1247 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1248 ; SI-NEXT: buffer_store_short v2, v[0:1], s[4:7], 0 addr64
1249 ; SI-NEXT: s_endpgm
1250 ;
1251 ; GFX8-LABEL: v_fdiv_f16_afn:
1252 ; GFX8: ; %bb.0: ; %entry
1253 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1254 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1255 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 1, v0
1256 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1257 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1258 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v4
1259 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1260 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
1261 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v4
1262 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
1263 ; GFX8-NEXT: flat_load_ushort v5, v[0:1] glc
1264 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1265 ; GFX8-NEXT: flat_load_ushort v0, v[2:3] glc
1266 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1267 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1268 ; GFX8-NEXT: v_rcp_f16_e32 v2, v0
1269 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v4
1270 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1271 ; GFX8-NEXT: v_mul_f16_e32 v2, v5, v2
1272 ; GFX8-NEXT: flat_store_short v[0:1], v2
1273 ; GFX8-NEXT: s_endpgm
1274 ;
1275 ; GFX9-LABEL: v_fdiv_f16_afn:
1276 ; GFX9: ; %bb.0: ; %entry
1277 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1278 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1279 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1280 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1281 ; GFX9-NEXT: global_load_ushort v1, v0, s[6:7] glc
1282 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1283 ; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] glc
1284 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1285 ; GFX9-NEXT: v_rcp_f16_e32 v2, v2
1286 ; GFX9-NEXT: v_mul_f16_e32 v1, v1, v2
1287 ; GFX9-NEXT: global_store_short v0, v1, s[4:5]
1288 ; GFX9-NEXT: s_endpgm
1289 ;
1290 ; GFX10-LABEL: v_fdiv_f16_afn:
1291 ; GFX10: ; %bb.0: ; %entry
1292 ; GFX10-NEXT: s_clause 0x1
1293 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1294 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1295 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1296 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1297 ; GFX10-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
1298 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1299 ; GFX10-NEXT: global_load_ushort v2, v0, s[2:3] glc dlc
1300 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1301 ; GFX10-NEXT: v_rcp_f16_e32 v2, v2
1302 ; GFX10-NEXT: v_mul_f16_e32 v1, v1, v2
1303 ; GFX10-NEXT: global_store_short v0, v1, s[4:5]
1304 ; GFX10-NEXT: s_endpgm
1305 ;
1306 ; GFX11-LABEL: v_fdiv_f16_afn:
1307 ; GFX11: ; %bb.0: ; %entry
1308 ; GFX11-NEXT: s_clause 0x1
1309 ; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1310 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1311 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1312 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1313 ; GFX11-NEXT: global_load_u16 v1, v0, s[6:7] glc dlc
1314 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1315 ; GFX11-NEXT: global_load_u16 v2, v0, s[0:1] glc dlc
1316 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1317 ; GFX11-NEXT: v_rcp_f16_e32 v2, v2
1318 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1319 ; GFX11-NEXT: v_mul_f16_e32 v1, v1, v2
1320 ; GFX11-NEXT: global_store_b16 v0, v1, s[4:5]
1321 ; GFX11-NEXT: s_nop 0
1322 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1323 ; GFX11-NEXT: s_endpgm
1324 entry:
1325 %tid = call i32 @llvm.amdgcn.workitem.id.x()
1326 %tid.ext = sext i32 %tid to i64
1327 %gep.a = getelementptr inbounds half, ptr addrspace(1) %a, i64 %tid.ext
1328 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
1329 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
1330 %a.val = load volatile half, ptr addrspace(1) %gep.a
1331 %b.val = load volatile half, ptr addrspace(1) %gep.b
1332 %r.val = fdiv afn half %a.val, %b.val
1333 store half %r.val, ptr addrspace(1) %gep.r
1334 ret void
1335 }
1337 define amdgpu_kernel void @v_fdiv_f16_unsafe(ptr addrspace(1) %r, ptr addrspace(1) %a, ptr addrspace(1) %b) #2 {
1338 ; SI-LABEL: v_fdiv_f16_unsafe:
1339 ; SI: ; %bb.0: ; %entry
1340 ; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1341 ; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xd
1342 ; SI-NEXT: s_mov_b32 s3, 0xf000
1343 ; SI-NEXT: s_mov_b32 s2, 0
1344 ; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1345 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1346 ; SI-NEXT: s_mov_b64 s[0:1], s[6:7]
1347 ; SI-NEXT: v_mov_b32_e32 v1, 0
1348 ; SI-NEXT: s_mov_b64 s[10:11], s[2:3]
1349 ; SI-NEXT: buffer_load_ushort v2, v[0:1], s[0:3], 0 addr64 glc
1350 ; SI-NEXT: s_waitcnt vmcnt(0)
1351 ; SI-NEXT: buffer_load_ushort v3, v[0:1], s[8:11], 0 addr64 glc
1352 ; SI-NEXT: s_waitcnt vmcnt(0)
1353 ; SI-NEXT: s_mov_b64 s[6:7], s[2:3]
1354 ; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
1355 ; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
1356 ; SI-NEXT: v_rcp_f32_e32 v3, v3
1357 ; SI-NEXT: v_mul_f32_e32 v2, v2, v3
1358 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
1359 ; SI-NEXT: buffer_store_short v2, v[0:1], s[4:7], 0 addr64
1360 ; SI-NEXT: s_endpgm
1361 ;
1362 ; GFX8-LABEL: v_fdiv_f16_unsafe:
1363 ; GFX8: ; %bb.0: ; %entry
1364 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1365 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1366 ; GFX8-NEXT: v_lshlrev_b32_e32 v4, 1, v0
1367 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1368 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1369 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v4
1370 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1371 ; GFX8-NEXT: v_mov_b32_e32 v3, s1
1372 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v4
1373 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
1374 ; GFX8-NEXT: flat_load_ushort v5, v[0:1] glc
1375 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1376 ; GFX8-NEXT: flat_load_ushort v0, v[2:3] glc
1377 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1378 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1379 ; GFX8-NEXT: v_rcp_f16_e32 v2, v0
1380 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v4
1381 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1382 ; GFX8-NEXT: v_mul_f16_e32 v2, v5, v2
1383 ; GFX8-NEXT: flat_store_short v[0:1], v2
1384 ; GFX8-NEXT: s_endpgm
1385 ;
1386 ; GFX9-LABEL: v_fdiv_f16_unsafe:
1387 ; GFX9: ; %bb.0: ; %entry
1388 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1389 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1390 ; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1391 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1392 ; GFX9-NEXT: global_load_ushort v1, v0, s[6:7] glc
1393 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1394 ; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] glc
1395 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1396 ; GFX9-NEXT: v_rcp_f16_e32 v2, v2
1397 ; GFX9-NEXT: v_mul_f16_e32 v1, v1, v2
1398 ; GFX9-NEXT: global_store_short v0, v1, s[4:5]
1399 ; GFX9-NEXT: s_endpgm
1400 ;
1401 ; GFX10-LABEL: v_fdiv_f16_unsafe:
1402 ; GFX10: ; %bb.0: ; %entry
1403 ; GFX10-NEXT: s_clause 0x1
1404 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1405 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1406 ; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1407 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1408 ; GFX10-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
1409 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1410 ; GFX10-NEXT: global_load_ushort v2, v0, s[2:3] glc dlc
1411 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1412 ; GFX10-NEXT: v_rcp_f16_e32 v2, v2
1413 ; GFX10-NEXT: v_mul_f16_e32 v1, v1, v2
1414 ; GFX10-NEXT: global_store_short v0, v1, s[4:5]
1415 ; GFX10-NEXT: s_endpgm
1416 ;
1417 ; GFX11-LABEL: v_fdiv_f16_unsafe:
1418 ; GFX11: ; %bb.0: ; %entry
1419 ; GFX11-NEXT: s_clause 0x1
1420 ; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1421 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1422 ; GFX11-NEXT: v_lshlrev_b32_e32 v0, 1, v0
1423 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1424 ; GFX11-NEXT: global_load_u16 v1, v0, s[6:7] glc dlc
1425 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1426 ; GFX11-NEXT: global_load_u16 v2, v0, s[0:1] glc dlc
1427 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1428 ; GFX11-NEXT: v_rcp_f16_e32 v2, v2
1429 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1430 ; GFX11-NEXT: v_mul_f16_e32 v1, v1, v2
1431 ; GFX11-NEXT: global_store_b16 v0, v1, s[4:5]
1432 ; GFX11-NEXT: s_nop 0
1433 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1434 ; GFX11-NEXT: s_endpgm
1435 entry:
1436 %tid = call i32 @llvm.amdgcn.workitem.id.x()
1437 %tid.ext = sext i32 %tid to i64
1438 %gep.a = getelementptr inbounds half, ptr addrspace(1) %a, i64 %tid.ext
1439 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext
1440 %gep.r = getelementptr inbounds half, ptr addrspace(1) %r, i64 %tid.ext
1441 %a.val = load volatile half, ptr addrspace(1) %gep.a
1442 %b.val = load volatile half, ptr addrspace(1) %gep.b
1443 %r.val = fdiv half %a.val, %b.val
1444 store half %r.val, ptr addrspace(1) %gep.r
1445 ret void
1446 }
1448 define amdgpu_kernel void @div_afn_2_x_pat_f16(ptr addrspace(1) %out) #0 {
1449 ; SI-LABEL: div_afn_2_x_pat_f16:
1450 ; SI: ; %bb.0:
1451 ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
1452 ; SI-NEXT: s_mov_b32 s3, 0xf000
1453 ; SI-NEXT: s_mov_b32 s2, -1
1454 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1455 ; SI-NEXT: buffer_load_ushort v0, off, s[0:3], 0
1456 ; SI-NEXT: s_waitcnt vmcnt(0)
1457 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1458 ; SI-NEXT: v_mul_f32_e32 v0, 0.5, v0
1459 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1460 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
1461 ; SI-NEXT: s_endpgm
1462 ;
1463 ; GFX8-LABEL: div_afn_2_x_pat_f16:
1464 ; GFX8: ; %bb.0:
1465 ; GFX8-NEXT: flat_load_ushort v0, v[0:1]
1466 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1467 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1468 ; GFX8-NEXT: v_mul_f16_e32 v2, 0.5, v0
1469 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1470 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1471 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1472 ; GFX8-NEXT: flat_store_short v[0:1], v2
1473 ; GFX8-NEXT: s_endpgm
1474 ;
1475 ; GFX9-LABEL: div_afn_2_x_pat_f16:
1476 ; GFX9: ; %bb.0:
1477 ; GFX9-NEXT: global_load_ushort v0, v[0:1], off
1478 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1479 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
1480 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1481 ; GFX9-NEXT: v_mul_f16_e32 v0, 0.5, v0
1482 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1483 ; GFX9-NEXT: global_store_short v1, v0, s[0:1]
1484 ; GFX9-NEXT: s_endpgm
1485 ;
1486 ; GFX10-LABEL: div_afn_2_x_pat_f16:
1487 ; GFX10: ; %bb.0:
1488 ; GFX10-NEXT: global_load_ushort v0, v[0:1], off
1489 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1490 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
1491 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1492 ; GFX10-NEXT: v_mul_f16_e32 v0, 0.5, v0
1493 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1494 ; GFX10-NEXT: global_store_short v1, v0, s[0:1]
1495 ; GFX10-NEXT: s_endpgm
1496 ;
1497 ; GFX11-LABEL: div_afn_2_x_pat_f16:
1498 ; GFX11: ; %bb.0:
1499 ; GFX11-NEXT: global_load_u16 v0, v[0:1], off
1500 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
1501 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
1502 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1503 ; GFX11-NEXT: v_mul_f16_e32 v0, 0.5, v0
1504 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1505 ; GFX11-NEXT: global_store_b16 v1, v0, s[0:1]
1506 ; GFX11-NEXT: s_nop 0
1507 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1508 ; GFX11-NEXT: s_endpgm
1509 %x = load half, ptr addrspace(1) undef
1510 %rcp = fdiv afn half %x, 2.0
1511 store half %rcp, ptr addrspace(1) %out, align 4
1512 ret void
1513 }
1515 define amdgpu_kernel void @div_afn_k_x_pat_f16(ptr addrspace(1) %out) #0 {
1516 ; SI-LABEL: div_afn_k_x_pat_f16:
1517 ; SI: ; %bb.0:
1518 ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
1519 ; SI-NEXT: s_mov_b32 s3, 0xf000
1520 ; SI-NEXT: s_mov_b32 s2, -1
1521 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1522 ; SI-NEXT: buffer_load_ushort v0, off, s[0:3], 0
1523 ; SI-NEXT: s_waitcnt vmcnt(0)
1524 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1525 ; SI-NEXT: v_mul_f32_e32 v0, 0x3dcccccd, v0
1526 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1527 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
1528 ; SI-NEXT: s_endpgm
1529 ;
1530 ; GFX8-LABEL: div_afn_k_x_pat_f16:
1531 ; GFX8: ; %bb.0:
1532 ; GFX8-NEXT: flat_load_ushort v0, v[0:1]
1533 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1534 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1535 ; GFX8-NEXT: v_mul_f16_e32 v2, 0x2e66, v0
1536 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1537 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1538 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1539 ; GFX8-NEXT: flat_store_short v[0:1], v2
1540 ; GFX8-NEXT: s_endpgm
1541 ;
1542 ; GFX9-LABEL: div_afn_k_x_pat_f16:
1543 ; GFX9: ; %bb.0:
1544 ; GFX9-NEXT: global_load_ushort v0, v[0:1], off
1545 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1546 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
1547 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1548 ; GFX9-NEXT: v_mul_f16_e32 v0, 0x2e66, v0
1549 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1550 ; GFX9-NEXT: global_store_short v1, v0, s[0:1]
1551 ; GFX9-NEXT: s_endpgm
1552 ;
1553 ; GFX10-LABEL: div_afn_k_x_pat_f16:
1554 ; GFX10: ; %bb.0:
1555 ; GFX10-NEXT: global_load_ushort v0, v[0:1], off
1556 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1557 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
1558 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1559 ; GFX10-NEXT: v_mul_f16_e32 v0, 0x2e66, v0
1560 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1561 ; GFX10-NEXT: global_store_short v1, v0, s[0:1]
1562 ; GFX10-NEXT: s_endpgm
1563 ;
1564 ; GFX11-LABEL: div_afn_k_x_pat_f16:
1565 ; GFX11: ; %bb.0:
1566 ; GFX11-NEXT: global_load_u16 v0, v[0:1], off
1567 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
1568 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
1569 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1570 ; GFX11-NEXT: v_mul_f16_e32 v0, 0x2e66, v0
1571 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1572 ; GFX11-NEXT: global_store_b16 v1, v0, s[0:1]
1573 ; GFX11-NEXT: s_nop 0
1574 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1575 ; GFX11-NEXT: s_endpgm
1576 %x = load half, ptr addrspace(1) undef
1577 %rcp = fdiv afn half %x, 10.0
1578 store half %rcp, ptr addrspace(1) %out, align 4
1579 ret void
1580 }
1582 define amdgpu_kernel void @div_afn_neg_k_x_pat_f16(ptr addrspace(1) %out) #0 {
1583 ; SI-LABEL: div_afn_neg_k_x_pat_f16:
1584 ; SI: ; %bb.0:
1585 ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
1586 ; SI-NEXT: s_mov_b32 s3, 0xf000
1587 ; SI-NEXT: s_mov_b32 s2, -1
1588 ; SI-NEXT: s_waitcnt lgkmcnt(0)
1589 ; SI-NEXT: buffer_load_ushort v0, off, s[0:3], 0
1590 ; SI-NEXT: s_waitcnt vmcnt(0)
1591 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1592 ; SI-NEXT: v_mul_f32_e32 v0, 0xbdcccccd, v0
1593 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1594 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
1595 ; SI-NEXT: s_endpgm
1596 ;
1597 ; GFX8-LABEL: div_afn_neg_k_x_pat_f16:
1598 ; GFX8: ; %bb.0:
1599 ; GFX8-NEXT: flat_load_ushort v0, v[0:1]
1600 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1601 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1602 ; GFX8-NEXT: v_mul_f16_e32 v2, 0xae66, v0
1603 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1604 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1605 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1606 ; GFX8-NEXT: flat_store_short v[0:1], v2
1607 ; GFX8-NEXT: s_endpgm
1608 ;
1609 ; GFX9-LABEL: div_afn_neg_k_x_pat_f16:
1610 ; GFX9: ; %bb.0:
1611 ; GFX9-NEXT: global_load_ushort v0, v[0:1], off
1612 ; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1613 ; GFX9-NEXT: v_mov_b32_e32 v1, 0
1614 ; GFX9-NEXT: s_waitcnt vmcnt(0)
1615 ; GFX9-NEXT: v_mul_f16_e32 v0, 0xae66, v0
1616 ; GFX9-NEXT: s_waitcnt lgkmcnt(0)
1617 ; GFX9-NEXT: global_store_short v1, v0, s[0:1]
1618 ; GFX9-NEXT: s_endpgm
1619 ;
1620 ; GFX10-LABEL: div_afn_neg_k_x_pat_f16:
1621 ; GFX10: ; %bb.0:
1622 ; GFX10-NEXT: global_load_ushort v0, v[0:1], off
1623 ; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
1624 ; GFX10-NEXT: v_mov_b32_e32 v1, 0
1625 ; GFX10-NEXT: s_waitcnt vmcnt(0)
1626 ; GFX10-NEXT: v_mul_f16_e32 v0, 0xae66, v0
1627 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
1628 ; GFX10-NEXT: global_store_short v1, v0, s[0:1]
1629 ; GFX10-NEXT: s_endpgm
1630 ;
1631 ; GFX11-LABEL: div_afn_neg_k_x_pat_f16:
1632 ; GFX11: ; %bb.0:
1633 ; GFX11-NEXT: global_load_u16 v0, v[0:1], off
1634 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
1635 ; GFX11-NEXT: v_mov_b32_e32 v1, 0
1636 ; GFX11-NEXT: s_waitcnt vmcnt(0)
1637 ; GFX11-NEXT: v_mul_f16_e32 v0, 0xae66, v0
1638 ; GFX11-NEXT: s_waitcnt lgkmcnt(0)
1639 ; GFX11-NEXT: global_store_b16 v1, v0, s[0:1]
1640 ; GFX11-NEXT: s_nop 0
1641 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1642 ; GFX11-NEXT: s_endpgm
1643 %x = load half, ptr addrspace(1) undef
1644 %rcp = fdiv afn half %x, -10.0
1645 store half %rcp, ptr addrspace(1) %out, align 4
1646 ret void
1647 }
1649 define half @v_fdiv_f16_arcp(half %x, half %y) {
1650 ; SI-LABEL: v_fdiv_f16_arcp:
1651 ; SI: ; %bb.0:
1652 ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1653 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1654 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
1655 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1656 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
1657 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1658 ; SI-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v0
1659 ; SI-NEXT: v_rcp_f32_e32 v3, v2
1660 ; SI-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
1661 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1662 ; SI-NEXT: v_fma_f32 v5, -v2, v3, 1.0
1663 ; SI-NEXT: v_fma_f32 v3, v5, v3, v3
1664 ; SI-NEXT: v_mul_f32_e32 v5, v4, v3
1665 ; SI-NEXT: v_fma_f32 v6, -v2, v5, v4
1666 ; SI-NEXT: v_fma_f32 v5, v6, v3, v5
1667 ; SI-NEXT: v_fma_f32 v2, -v2, v5, v4
1668 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1669 ; SI-NEXT: v_div_fmas_f32 v2, v2, v3, v5
1670 ; SI-NEXT: v_div_fixup_f32 v0, v2, v1, v0
1671 ; SI-NEXT: s_setpc_b64 s[30:31]
1672 ;
1673 ; GFX8-LABEL: v_fdiv_f16_arcp:
1674 ; GFX8: ; %bb.0:
1675 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1676 ; GFX8-NEXT: v_rcp_f16_e32 v1, v1
1677 ; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
1678 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1679 ;
1680 ; GFX9-LABEL: v_fdiv_f16_arcp:
1681 ; GFX9: ; %bb.0:
1682 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1683 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
1684 ; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
1685 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1686 ;
1687 ; GFX10-LABEL: v_fdiv_f16_arcp:
1688 ; GFX10: ; %bb.0:
1689 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1690 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
1691 ; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
1692 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1693 ;
1694 ; GFX11-LABEL: v_fdiv_f16_arcp:
1695 ; GFX11: ; %bb.0:
1696 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1697 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
1698 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1699 ; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
1700 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1701 %fdiv = fdiv arcp half %x, %y
1702 ret half %fdiv
1703 }
1705 define half @v_fdiv_f16_afn_nsz(half %x, half %y) {
1706 ; SI-LABEL: v_fdiv_f16_afn_nsz:
1707 ; SI: ; %bb.0:
1708 ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1709 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
1710 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1711 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
1712 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1713 ; SI-NEXT: v_rcp_f32_e32 v1, v1
1714 ; SI-NEXT: v_mul_f32_e32 v0, v0, v1
1715 ; SI-NEXT: s_setpc_b64 s[30:31]
1716 ;
1717 ; GFX8-LABEL: v_fdiv_f16_afn_nsz:
1718 ; GFX8: ; %bb.0:
1719 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1720 ; GFX8-NEXT: v_rcp_f16_e32 v1, v1
1721 ; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
1722 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1723 ;
1724 ; GFX9-LABEL: v_fdiv_f16_afn_nsz:
1725 ; GFX9: ; %bb.0:
1726 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1727 ; GFX9-NEXT: v_rcp_f16_e32 v1, v1
1728 ; GFX9-NEXT: v_mul_f16_e32 v0, v0, v1
1729 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1730 ;
1731 ; GFX10-LABEL: v_fdiv_f16_afn_nsz:
1732 ; GFX10: ; %bb.0:
1733 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1734 ; GFX10-NEXT: v_rcp_f16_e32 v1, v1
1735 ; GFX10-NEXT: v_mul_f16_e32 v0, v0, v1
1736 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1737 ;
1738 ; GFX11-LABEL: v_fdiv_f16_afn_nsz:
1739 ; GFX11: ; %bb.0:
1740 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1741 ; GFX11-NEXT: v_rcp_f16_e32 v1, v1
1742 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1743 ; GFX11-NEXT: v_mul_f16_e32 v0, v0, v1
1744 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1745 %fdiv = fdiv afn nsz half %x, %y
1746 ret half %fdiv
1747 }
1749 define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
1750 ; GFX6-IEEE-LABEL: v_rsq_v2f16:
1751 ; GFX6-IEEE: ; %bb.0:
1752 ; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1753 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
1754 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
1755 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, 1.0
1756 ; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
1757 ; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
1758 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
1759 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
1760 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
1761 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
1762 ; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v0, v0, v2
1763 ; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v3
1764 ; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v2, v0, v2
1765 ; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v1, v1, v2
1766 ; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v6, 1.0
1767 ; GFX6-IEEE-NEXT: v_fma_f32 v6, v9, v6, v6
1768 ; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v4, v6
1769 ; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v5
1770 ; GFX6-IEEE-NEXT: v_fma_f32 v10, -v3, v9, v4
1771 ; GFX6-IEEE-NEXT: v_fma_f32 v9, v10, v6, v9
1772 ; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v9, v4
1773 ; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v6, v9
1774 ; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v3, v0, v2
1775 ; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v8, 1.0
1776 ; GFX6-IEEE-NEXT: v_div_scale_f32 v7, s[4:5], v2, v1, v2
1777 ; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v8, v8
1778 ; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v7, v3
1779 ; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v7
1780 ; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
1781 ; GFX6-IEEE-NEXT: v_fma_f32 v5, -v5, v4, v7
1782 ; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
1783 ; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v5, v3, v4
1784 ; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, v2
1785 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
1786 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
1787 ; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
1788 ;
1789 ; GFX6-FLUSH-LABEL: v_rsq_v2f16:
1790 ; GFX6-FLUSH: ; %bb.0:
1791 ; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1792 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
1793 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
1794 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, 1.0
1795 ; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
1796 ; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v1, v1
1797 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1798 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
1799 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
1800 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
1801 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[4:5], v0, v0, v2
1802 ; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v4, v3
1803 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v5, vcc, v2, v0, v2
1804 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1805 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v3, v4, 1.0
1806 ; GFX6-FLUSH-NEXT: v_fma_f32 v4, v6, v4, v4
1807 ; GFX6-FLUSH-NEXT: v_mul_f32_e32 v6, v5, v4
1808 ; GFX6-FLUSH-NEXT: v_fma_f32 v7, -v3, v6, v5
1809 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, v7, v4, v6
1810 ; GFX6-FLUSH-NEXT: v_fma_f32 v3, -v3, v6, v5
1811 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1812 ; GFX6-FLUSH-NEXT: v_div_fmas_f32 v3, v3, v4, v6
1813 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, 1.0
1814 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
1815 ; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v3, v0, v2
1816 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
1817 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v4
1818 ; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
1819 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v5, vcc, v4, v1, v4
1820 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1821 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v2, v3, 1.0
1822 ; GFX6-FLUSH-NEXT: v_fma_f32 v3, v6, v3, v3
1823 ; GFX6-FLUSH-NEXT: v_mul_f32_e32 v6, v5, v3
1824 ; GFX6-FLUSH-NEXT: v_fma_f32 v7, -v2, v6, v5
1825 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, v7, v3, v6
1826 ; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v6, v5
1827 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1828 ; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v6
1829 ; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v2, v1, v4
1830 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
1831 ; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
1832 ;
1833 ; SI-LABEL: v_rsq_v2f16:
1834 ; SI: ; %bb.0:
1835 ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1836 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1837 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1838 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
1839 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1840 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
1841 ; SI-NEXT: v_sqrt_f32_e32 v0, v0
1842 ; SI-NEXT: v_sqrt_f32_e32 v1, v1
1843 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
1844 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
1845 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
1846 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
1847 ; SI-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
1848 ; SI-NEXT: v_rcp_f32_e32 v3, v2
1849 ; SI-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
1850 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1851 ; SI-NEXT: v_fma_f32 v5, -v2, v3, 1.0
1852 ; SI-NEXT: v_fma_f32 v3, v5, v3, v3
1853 ; SI-NEXT: v_mul_f32_e32 v5, v4, v3
1854 ; SI-NEXT: v_fma_f32 v6, -v2, v5, v4
1855 ; SI-NEXT: v_fma_f32 v5, v6, v3, v5
1856 ; SI-NEXT: v_fma_f32 v2, -v2, v5, v4
1857 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1858 ; SI-NEXT: v_div_fmas_f32 v2, v2, v3, v5
1859 ; SI-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
1860 ; SI-NEXT: v_rcp_f32_e32 v4, v3
1861 ; SI-NEXT: v_div_scale_f32 v5, vcc, 1.0, v1, 1.0
1862 ; SI-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
1863 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
1864 ; SI-NEXT: v_fma_f32 v2, -v3, v4, 1.0
1865 ; SI-NEXT: v_fma_f32 v2, v2, v4, v4
1866 ; SI-NEXT: v_mul_f32_e32 v4, v5, v2
1867 ; SI-NEXT: v_fma_f32 v6, -v3, v4, v5
1868 ; SI-NEXT: v_fma_f32 v4, v6, v2, v4
1869 ; SI-NEXT: v_fma_f32 v3, -v3, v4, v5
1870 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
1871 ; SI-NEXT: v_div_fmas_f32 v2, v3, v2, v4
1872 ; SI-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
1873 ; SI-NEXT: s_setpc_b64 s[30:31]
1874 ;
1875 ; GFX8-LABEL: v_rsq_v2f16:
1876 ; GFX8: ; %bb.0:
1877 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1878 ; GFX8-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
1879 ; GFX8-NEXT: v_rsq_f16_e32 v0, v0
1880 ; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
1881 ; GFX8-NEXT: s_setpc_b64 s[30:31]
1882 ;
1883 ; GFX9-LABEL: v_rsq_v2f16:
1884 ; GFX9: ; %bb.0:
1885 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1886 ; GFX9-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
1887 ; GFX9-NEXT: v_rsq_f16_e32 v0, v0
1888 ; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
1889 ; GFX9-NEXT: s_setpc_b64 s[30:31]
1890 ;
1891 ; GFX10-LABEL: v_rsq_v2f16:
1892 ; GFX10: ; %bb.0:
1893 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1894 ; GFX10-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
1895 ; GFX10-NEXT: v_rsq_f16_e32 v0, v0
1896 ; GFX10-NEXT: v_pack_b32_f16 v0, v0, v1
1897 ; GFX10-NEXT: s_setpc_b64 s[30:31]
1898 ;
1899 ; GFX11-LABEL: v_rsq_v2f16:
1900 ; GFX11: ; %bb.0:
1901 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1902 ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
1903 ; GFX11-NEXT: v_rsq_f16_e32 v0, v0
1904 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
1905 ; GFX11-NEXT: v_rsq_f16_e32 v1, v1
1906 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
1907 ; GFX11-NEXT: v_pack_b32_f16 v0, v0, v1
1908 ; GFX11-NEXT: s_setpc_b64 s[30:31]
1909 ; GFX9-IEEE-LABEL: v_rsq_v2f16:
1910 ; GFX9-IEEE: ; %bb.0:
1911 ; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1912 ; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
1913 ; GFX9-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
1914 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, 1.0
1915 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
1916 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
1917 ; GFX9-IEEE-NEXT: v_rcp_f32_e32 v2, v2
1918 ; GFX9-IEEE-NEXT: v_rcp_f32_e32 v3, v3
1919 ; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
1920 ; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
1921 ; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
1922 ; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
1923 ; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
1924 ; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
1925 ; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
1926 ; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
1927 ; GFX9-FLUSH-LABEL: v_rsq_v2f16:
1928 ; GFX9-FLUSH: ; %bb.0:
1929 ; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1930 ; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
1931 ; GFX9-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
1932 ; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
1933 ; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
1934 ; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
1935 ; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
1936 ; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, 1.0, v2, 0 op_sel_hi:[1,0,0]
1937 ; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v3, 1.0, v3, 0 op_sel_hi:[1,0,0]
1938 ; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
1939 ; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
1940 ; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
1941 ; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
1942 %sqrt = call contract <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
1943 %fdiv = fdiv contract <2 x half> <half 1.0, half 1.0>, %sqrt
1944 ret <2 x half> %fdiv
1945 }
1947 define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
1948 ; GFX6-IEEE-LABEL: v_neg_rsq_v2f16:
1949 ; GFX6-IEEE: ; %bb.0:
1950 ; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1951 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
1952 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
1953 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, -1.0
1954 ; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
1955 ; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
1956 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
1957 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
1958 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
1959 ; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
1960 ; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v0, v0, v2
1961 ; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v3
1962 ; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v2, v0, v2
1963 ; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v1, v1, v2
1964 ; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v6, 1.0
1965 ; GFX6-IEEE-NEXT: v_fma_f32 v6, v9, v6, v6
1966 ; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v4, v6
1967 ; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v5
1968 ; GFX6-IEEE-NEXT: v_fma_f32 v10, -v3, v9, v4
1969 ; GFX6-IEEE-NEXT: v_fma_f32 v9, v10, v6, v9
1970 ; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v9, v4
1971 ; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v6, v9
1972 ; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v3, v0, v2
1973 ; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v8, 1.0
1974 ; GFX6-IEEE-NEXT: v_div_scale_f32 v7, s[4:5], v2, v1, v2
1975 ; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v8, v8
1976 ; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v7, v3
1977 ; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v7
1978 ; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
1979 ; GFX6-IEEE-NEXT: v_fma_f32 v5, -v5, v4, v7
1980 ; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
1981 ; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v5, v3, v4
1982 ; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, v2
1983 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
1984 ; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
1985 ; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
1986 ;
1987 ; GFX6-FLUSH-LABEL: v_neg_rsq_v2f16:
1988 ; GFX6-FLUSH: ; %bb.0:
1989 ; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
1990 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
1991 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
1992 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, -1.0
1993 ; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
1994 ; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v1, v1
1995 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
1996 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
1997 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
1998 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
1999 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[4:5], v0, v0, v2
2000 ; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v4, v3
2001 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v5, vcc, v2, v0, v2
2002 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
2003 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v3, v4, 1.0
2004 ; GFX6-FLUSH-NEXT: v_fma_f32 v4, v6, v4, v4
2005 ; GFX6-FLUSH-NEXT: v_mul_f32_e32 v6, v5, v4
2006 ; GFX6-FLUSH-NEXT: v_fma_f32 v7, -v3, v6, v5
2007 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, v7, v4, v6
2008 ; GFX6-FLUSH-NEXT: v_fma_f32 v3, -v3, v6, v5
2009 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
2010 ; GFX6-FLUSH-NEXT: v_div_fmas_f32 v3, v3, v4, v6
2011 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, -1.0
2012 ; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
2013 ; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v3, v0, v2
2014 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
2015 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, v4
2016 ; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
2017 ; GFX6-FLUSH-NEXT: v_div_scale_f32 v5, vcc, v4, v1, v4
2018 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
2019 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v2, v3, 1.0
2020 ; GFX6-FLUSH-NEXT: v_fma_f32 v3, v6, v3, v3
2021 ; GFX6-FLUSH-NEXT: v_mul_f32_e32 v6, v5, v3
2022 ; GFX6-FLUSH-NEXT: v_fma_f32 v7, -v2, v6, v5
2023 ; GFX6-FLUSH-NEXT: v_fma_f32 v6, v7, v3, v6
2024 ; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v6, v5
2025 ; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
2026 ; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v6
2027 ; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v2, v1, v4
2028 ; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
2029 ; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
2030 ;
2031 ; SI-LABEL: v_neg_rsq_v2f16:
2032 ; SI: ; %bb.0:
2033 ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2034 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
2035 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
2036 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
2037 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
2038 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
2039 ; SI-NEXT: v_sqrt_f32_e32 v0, v0
2040 ; SI-NEXT: v_sqrt_f32_e32 v1, v1
2041 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
2042 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
2043 ; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
2044 ; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
2045 ; SI-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, -1.0
2046 ; SI-NEXT: v_rcp_f32_e32 v3, v2
2047 ; SI-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
2048 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
2049 ; SI-NEXT: v_fma_f32 v5, -v2, v3, 1.0
2050 ; SI-NEXT: v_fma_f32 v3, v5, v3, v3
2051 ; SI-NEXT: v_mul_f32_e32 v5, v4, v3
2052 ; SI-NEXT: v_fma_f32 v6, -v2, v5, v4
2053 ; SI-NEXT: v_fma_f32 v5, v6, v3, v5
2054 ; SI-NEXT: v_fma_f32 v2, -v2, v5, v4
2055 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
2056 ; SI-NEXT: v_div_fmas_f32 v2, v2, v3, v5
2057 ; SI-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, -1.0
2058 ; SI-NEXT: v_rcp_f32_e32 v4, v3
2059 ; SI-NEXT: v_div_scale_f32 v5, vcc, -1.0, v1, -1.0
2060 ; SI-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
2061 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
2062 ; SI-NEXT: v_fma_f32 v2, -v3, v4, 1.0
2063 ; SI-NEXT: v_fma_f32 v2, v2, v4, v4
2064 ; SI-NEXT: v_mul_f32_e32 v4, v5, v2
2065 ; SI-NEXT: v_fma_f32 v6, -v3, v4, v5
2066 ; SI-NEXT: v_fma_f32 v4, v6, v2, v4
2067 ; SI-NEXT: v_fma_f32 v3, -v3, v4, v5
2068 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
2069 ; SI-NEXT: v_div_fmas_f32 v2, v3, v2, v4
2070 ; SI-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
2071 ; SI-NEXT: s_setpc_b64 s[30:31]
2072 ;
2073 ; GFX8-LABEL: v_neg_rsq_v2f16:
2074 ; GFX8: ; %bb.0:
2075 ; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2076 ; GFX8-NEXT: v_rsq_f16_e32 v1, v0
2077 ; GFX8-NEXT: v_rsq_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
2078 ; GFX8-NEXT: v_mov_b32_e32 v2, 0x8000
2079 ; GFX8-NEXT: v_xor_b32_e32 v1, 0x8000, v1
2080 ; GFX8-NEXT: v_xor_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
2081 ; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2082 ; GFX8-NEXT: s_setpc_b64 s[30:31]
2083 ;
2084 ; GFX9-LABEL: v_neg_rsq_v2f16:
2085 ; GFX9: ; %bb.0:
2086 ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2087 ; GFX9-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
2088 ; GFX9-NEXT: v_rsq_f16_e32 v0, v0
2089 ; GFX9-NEXT: v_pack_b32_f16 v0, -v0, -v1
2090 ; GFX9-NEXT: s_setpc_b64 s[30:31]
2091 ;
2092 ; GFX10-LABEL: v_neg_rsq_v2f16:
2093 ; GFX10: ; %bb.0:
2094 ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2095 ; GFX10-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
2096 ; GFX10-NEXT: v_rsq_f16_e32 v0, v0
2097 ; GFX10-NEXT: v_pack_b32_f16 v0, -v0, -v1
2098 ; GFX10-NEXT: s_setpc_b64 s[30:31]
2099 ;
2100 ; GFX11-LABEL: v_neg_rsq_v2f16:
2101 ; GFX11: ; %bb.0:
2102 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2103 ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0
2104 ; GFX11-NEXT: v_rsq_f16_e32 v0, v0
2105 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
2106 ; GFX11-NEXT: v_rsq_f16_e32 v1, v1
2107 ; GFX11-NEXT: s_waitcnt_depctr 0xfff
2108 ; GFX11-NEXT: v_pack_b32_f16 v0, -v0, -v1
2109 ; GFX11-NEXT: s_setpc_b64 s[30:31]
2110 ; GFX9-IEEE-LABEL: v_neg_rsq_v2f16:
2111 ; GFX9-IEEE: ; %bb.0:
2112 ; GFX9-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2113 ; GFX9-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
2114 ; GFX9-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
2115 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v4, -1.0
2116 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
2117 ; GFX9-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
2118 ; GFX9-IEEE-NEXT: v_rcp_f32_e32 v2, v2
2119 ; GFX9-IEEE-NEXT: v_rcp_f32_e32 v3, v3
2120 ; GFX9-IEEE-NEXT: v_mul_f32_e32 v2, v4, v2
2121 ; GFX9-IEEE-NEXT: v_mul_f32_e32 v3, v4, v3
2122 ; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
2123 ; GFX9-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
2124 ; GFX9-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
2125 ; GFX9-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
2126 ; GFX9-IEEE-NEXT: v_pack_b32_f16 v0, v1, v0
2127 ; GFX9-IEEE-NEXT: s_setpc_b64 s[30:31]
2128 ; GFX9-FLUSH-LABEL: v_neg_rsq_v2f16:
2129 ; GFX9-FLUSH: ; %bb.0:
2130 ; GFX9-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
2131 ; GFX9-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
2132 ; GFX9-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
2133 ; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
2134 ; GFX9-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
2135 ; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v2, v2
2136 ; GFX9-FLUSH-NEXT: v_rcp_f32_e32 v3, v3
2137 ; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v2, -1.0, v2, 0 op_sel_hi:[1,0,0]
2138 ; GFX9-FLUSH-NEXT: v_mad_mixlo_f16 v3, -1.0, v3, 0 op_sel_hi:[1,0,0]
2139 ; GFX9-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
2140 ; GFX9-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
2141 ; GFX9-FLUSH-NEXT: v_pack_b32_f16 v0, v1, v0
2142 ; GFX9-FLUSH-NEXT: s_setpc_b64 s[30:31]
2143 %sqrt = call contract <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
2144 %fdiv = fdiv contract <2 x half> <half -1.0, half -1.0>, %sqrt
2145 ret <2 x half> %fdiv
2146 }
2148 declare i32 @llvm.amdgcn.workitem.id.x() #2
2149 declare half @llvm.sqrt.f16(half) #2
2150 declare half @llvm.fabs.f16(half) #2
2151 declare <2 x half> @llvm.sqrt.v2f16(<2 x half>) #2
2153 attributes #0 = { nounwind }
2154 attributes #1 = { nounwind readnone }
2155 attributes #2 = { nounwind "unsafe-fp-math"="true" }