1 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
2 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
3 ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
5 ; GCN-LABEL: mixlo_simple:
7 ; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2{{$}}
8 ; GFX9-NEXT: s_setpc_b64
11 ; CIVI: v_cvt_f16_f32_e32
12 define half @mixlo_simple(float %src0, float %src1, float %src2) #0 {
13 %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
14 %cvt.result = fptrunc float %result to half
18 ; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f16lo:
19 ; GFX9: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]{{$}}
22 define half @v_mad_mixlo_f16_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
23 %src0.ext = fpext half %src0 to float
24 %src1.ext = fpext half %src1 to float
25 %src2.ext = fpext half %src2 to float
26 %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
27 %cvt.result = fptrunc float %result to half
31 ; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32:
33 ; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
34 ; GFX9-NEXT: s_setpc_b64
37 define half @v_mad_mixlo_f16_f16lo_f16lo_f32(half %src0, half %src1, float %src2) #0 {
38 %src0.ext = fpext half %src0 to float
39 %src1.ext = fpext half %src1 to float
40 %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
41 %cvt.result = fptrunc float %result to half
45 ; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
47 ; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp{{$}}
48 ; GFX9-NEXT: s_setpc_b64
50 ; CIVI: v_mac_f32_e32 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]$}}
51 define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt(half %src0, half %src1, float %src2) #0 {
52 %src0.ext = fpext half %src0 to float
53 %src1.ext = fpext half %src1 to float
54 %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
55 %cvt.result = fptrunc float %result to half
56 %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
57 %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
61 ; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
63 ; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp{{$}}
64 ; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
65 ; GFX9-NEXT: s_setpc_b64
67 ; CIVI: v_mac_f32_e64 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
68 define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt(half %src0, half %src1, float %src2) #0 {
69 %src0.ext = fpext half %src0 to float
70 %src1.ext = fpext half %src1 to float
71 %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
72 %max = call float @llvm.maxnum.f32(float %result, float 0.0)
73 %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
74 %cvt.result = fptrunc float %clamp to half
78 ; FIXME: Should abe able to avoid extra register because first
79 ; operation only clobbers relevant lane.
80 ; GCN-LABEL: {{^}}v_mad_mix_v2f32:
82 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]{{$}}
83 ; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]{{$}}
84 ; GFX9-NEXT: v_mov_b32_e32 v0, v3
85 ; GFX9-NEXT: s_setpc_b64
86 define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
87 %src0.ext = fpext <2 x half> %src0 to <2 x float>
88 %src1.ext = fpext <2 x half> %src1 to <2 x float>
89 %src2.ext = fpext <2 x half> %src2 to <2 x float>
90 %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
91 %cvt.result = fptrunc <2 x float> %result to <2 x half>
92 ret <2 x half> %cvt.result
95 ; GCN-LABEL: {{^}}v_mad_mix_v3f32:
97 ; GFX9-NEXT: v_mad_mixlo_f16 v1, v1, v3, v5 op_sel_hi:[1,1,1]
98 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v2, v4 op_sel_hi:[1,1,1]
99 ; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
100 ; GFX9-NEXT: v_mov_b32_e32 v0, v3
101 ; GFX9-NEXT: s_setpc_b64
102 define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
103 %src0.ext = fpext <3 x half> %src0 to <3 x float>
104 %src1.ext = fpext <3 x half> %src1 to <3 x float>
105 %src2.ext = fpext <3 x half> %src2 to <3 x float>
106 %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
107 %cvt.result = fptrunc <3 x float> %result to <3 x half>
108 ret <3 x half> %cvt.result
111 ; GCN-LABEL: {{^}}v_mad_mix_v4f32:
113 ; GFX9-NEXT: v_mad_mixlo_f16 v6, v1, v3, v5 op_sel_hi:[1,1,1]
114 ; GFX9-NEXT: v_mad_mixlo_f16 v7, v0, v2, v4 op_sel_hi:[1,1,1]
115 ; GFX9-NEXT: v_mad_mixhi_f16 v7, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
116 ; GFX9-NEXT: v_mad_mixhi_f16 v6, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
117 ; GFX9-NEXT: v_mov_b32_e32 v0, v7
118 ; GFX9-NEXT: v_mov_b32_e32 v1, v6
119 ; GFX9-NEXT: s_setpc_b64
120 define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
121 %src0.ext = fpext <4 x half> %src0 to <4 x float>
122 %src1.ext = fpext <4 x half> %src1 to <4 x float>
123 %src2.ext = fpext <4 x half> %src2 to <4 x float>
124 %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
125 %cvt.result = fptrunc <4 x float> %result to <4 x half>
126 ret <4 x half> %cvt.result
130 ; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt:
131 ; GFX9: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp{{$}}
132 ; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp{{$}}
133 ; GFX9-NEXT: v_mov_b32_e32 v0, v3
134 ; GFX9-NEXT: s_setpc_b64
135 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
136 %src0.ext = fpext <2 x half> %src0 to <2 x float>
137 %src1.ext = fpext <2 x half> %src1 to <2 x float>
138 %src2.ext = fpext <2 x half> %src2 to <2 x float>
139 %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
140 %cvt.result = fptrunc <2 x float> %result to <2 x half>
141 %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %cvt.result, <2 x half> zeroinitializer)
142 %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
143 ret <2 x half> %clamp
146 ; FIXME: Should be packed into 2 registers per argument?
147 ; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_postcvt:
149 ; GFX9-DAG: v_mad_mixlo_f16 v{{[0-9]+}}, v0, v2, v4 op_sel_hi:[1,1,1] clamp
150 ; GFX9-DAG: v_mad_mixhi_f16 v{{[0-9]+}}, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
151 ; GFX9-DAG: v_mad_mixlo_f16 v{{[0-9]+}}, v1, v3, v5 op_sel_hi:[1,1,1]
152 ; GFX9-DAG: v_pk_max_f16 v1, v1, v1 clamp
153 ; GFX9: v_mov_b32_e32 v0, v{{[0-9]+}}
154 ; GFX9-NEXT: s_setpc_b64
155 define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
156 %src0.ext = fpext <3 x half> %src0 to <3 x float>
157 %src1.ext = fpext <3 x half> %src1 to <3 x float>
158 %src2.ext = fpext <3 x half> %src2 to <3 x float>
159 %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
160 %cvt.result = fptrunc <3 x float> %result to <3 x half>
161 %max = call <3 x half> @llvm.maxnum.v3f16(<3 x half> %cvt.result, <3 x half> zeroinitializer)
162 %clamp = call <3 x half> @llvm.minnum.v3f16(<3 x half> %max, <3 x half> <half 1.0, half 1.0, half 1.0>)
163 ret <3 x half> %clamp
166 ; GCN-LABEL: {{^}}v_mad_mix_v4f32_clamp_postcvt:
168 ; GFX9-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp
169 ; GFX9-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
170 ; GFX9-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp
171 ; GFX9-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
172 ; GFX9-NEXT: v_mov_b32_e32 v0, v6
173 ; GFX9-NEXT: v_mov_b32_e32 v1, v2
174 ; GFX9-NEXT: s_setpc_b64
175 define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
176 %src0.ext = fpext <4 x half> %src0 to <4 x float>
177 %src1.ext = fpext <4 x half> %src1 to <4 x float>
178 %src2.ext = fpext <4 x half> %src2 to <4 x float>
179 %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
180 %cvt.result = fptrunc <4 x float> %result to <4 x half>
181 %max = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %cvt.result, <4 x half> zeroinitializer)
182 %clamp = call <4 x half> @llvm.minnum.v4f16(<4 x half> %max, <4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>)
183 ret <4 x half> %clamp
186 ; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt_lo:
188 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
189 ; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
190 ; GFX9-NEXT: v_mov_b32_e32 v0, v3
191 ; GFX9-NEXT: s_setpc_b64
192 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
193 %src0.ext = fpext <2 x half> %src0 to <2 x float>
194 %src1.ext = fpext <2 x half> %src1 to <2 x float>
195 %src2.ext = fpext <2 x half> %src2 to <2 x float>
196 %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
197 %cvt.result = fptrunc <2 x float> %result to <2 x half>
198 %cvt.lo = extractelement <2 x half> %cvt.result, i32 0
199 %max.lo = call half @llvm.maxnum.f16(half %cvt.lo, half 0.0)
200 %clamp.lo = call half @llvm.minnum.f16(half %max.lo, half 1.0)
201 %insert = insertelement <2 x half> %cvt.result, half %clamp.lo, i32 0
202 ret <2 x half> %insert
205 ; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt_hi:
207 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
208 ; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
209 ; GFX9-NEXT: v_mov_b32_e32 v0, v3
210 ; GFX9-NEXT: s_setpc_b64
211 define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
212 %src0.ext = fpext <2 x half> %src0 to <2 x float>
213 %src1.ext = fpext <2 x half> %src1 to <2 x float>
214 %src2.ext = fpext <2 x half> %src2 to <2 x float>
215 %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
216 %cvt.result = fptrunc <2 x float> %result to <2 x half>
217 %cvt.hi = extractelement <2 x half> %cvt.result, i32 1
218 %max.hi = call half @llvm.maxnum.f16(half %cvt.hi, half 0.0)
219 %clamp.hi = call half @llvm.minnum.f16(half %max.hi, half 1.0)
220 %insert = insertelement <2 x half> %cvt.result, half %clamp.hi, i32 1
221 ret <2 x half> %insert
224 ; FIXME: Should be able to use mixlo/mixhi
225 ; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_precvt:
226 ; GFX9: v_mad_mix_f32 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
227 ; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
228 ; GFX9: v_cvt_f16_f32_e32 v1, v3
229 ; GFX9: v_cvt_f16_f32_e32 v0, v0
230 ; GFX9: v_and_b32_e32 v1, 0xffff, v1
231 ; GFX9: v_lshl_or_b32 v0, v0, 16, v1
233 define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
234 %src0.ext = fpext <2 x half> %src0 to <2 x float>
235 %src1.ext = fpext <2 x half> %src1 to <2 x float>
236 %src2.ext = fpext <2 x half> %src2 to <2 x float>
237 %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
238 %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %result, <2 x float> zeroinitializer)
239 %clamp = call <2 x float> @llvm.minnum.v2f32(<2 x float> %max, <2 x float> <float 1.0, float 1.0>)
240 %cvt.result = fptrunc <2 x float> %clamp to <2 x half>
241 ret <2 x half> %cvt.result
244 ; FIXME: Handling undef 4th component
245 ; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_precvt:
247 ; GFX9-NEXT: v_mad_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
248 ; GFX9-NEXT: v_mad_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
249 ; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
250 ; GFX9-NEXT: v_mad_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
251 ; GFX9-NEXT: v_cvt_f16_f32_e32 v2, v6
252 ; GFX9-NEXT: v_cvt_f16_f32_e32 v1, v1
253 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
254 ; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
255 ; GFX9-NEXT: s_setpc_b64
256 define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
257 %src0.ext = fpext <3 x half> %src0 to <3 x float>
258 %src1.ext = fpext <3 x half> %src1 to <3 x float>
259 %src2.ext = fpext <3 x half> %src2 to <3 x float>
260 %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
261 %max = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %result, <3 x float> zeroinitializer)
262 %clamp = call <3 x float> @llvm.minnum.v3f32(<3 x float> %max, <3 x float> <float 1.0, float 1.0, float 1.0>)
263 %cvt.result = fptrunc <3 x float> %clamp to <3 x half>
264 ret <3 x half> %cvt.result
267 ; GCN-LABEL: {{^}}v_mad_mix_v4f32_clamp_precvt:
268 ; GFX9: v_mad_mix_f32 v6, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
269 ; GFX9: v_mad_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
270 ; GFX9: v_mad_mix_f32 v3, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
271 ; GFX9: v_mad_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
273 ; GFX9: v_cvt_f16_f32
274 ; GFX9: v_cvt_f16_f32
275 ; GFX9: v_cvt_f16_f32
276 ; GFX9: v_cvt_f16_f32
277 define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
278 %src0.ext = fpext <4 x half> %src0 to <4 x float>
279 %src1.ext = fpext <4 x half> %src1 to <4 x float>
280 %src2.ext = fpext <4 x half> %src2 to <4 x float>
281 %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
282 %max = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %result, <4 x float> zeroinitializer)
283 %clamp = call <4 x float> @llvm.minnum.v4f32(<4 x float> %max, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>)
284 %cvt.result = fptrunc <4 x float> %clamp to <4 x half>
285 ret <4 x half> %cvt.result
288 declare half @llvm.minnum.f16(half, half) #1
289 declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
290 declare <3 x half> @llvm.minnum.v3f16(<3 x half>, <3 x half>) #1
291 declare <4 x half> @llvm.minnum.v4f16(<4 x half>, <4 x half>) #1
293 declare half @llvm.maxnum.f16(half, half) #1
294 declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
295 declare <3 x half> @llvm.maxnum.v3f16(<3 x half>, <3 x half>) #1
296 declare <4 x half> @llvm.maxnum.v4f16(<4 x half>, <4 x half>) #1
298 declare float @llvm.minnum.f32(float, float) #1
299 declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1
300 declare <3 x float> @llvm.minnum.v3f32(<3 x float>, <3 x float>) #1
301 declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) #1
303 declare float @llvm.maxnum.f32(float, float) #1
304 declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1
305 declare <3 x float> @llvm.maxnum.v3f32(<3 x float>, <3 x float>) #1
306 declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>) #1
308 declare float @llvm.fmuladd.f32(float, float, float) #1
309 declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
310 declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>) #1
311 declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #1
313 attributes #0 = { nounwind }
314 attributes #1 = { nounwind readnone speculatable }