?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Run DCE after a LoopFlatten test to reduce spurious output [nfc]
[llvm-project.git] / llvm / lib / CodeGen / SelectionDAG / LegalizeFloatTypes.cpp
blob1c429546a8a738dbe9dc15bdc28ff082f8ff0406
1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58 #ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61 #endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::STRICT_FCEIL:
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::STRICT_FCOS:
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::STRICT_FDIV:
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::STRICT_FEXP:
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::STRICT_FEXP2:
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::STRICT_FFLOOR:
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::STRICT_FLOG:
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::STRICT_FLOG2:
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::STRICT_FLOG10:
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::STRICT_FNEARBYINT:
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::STRICT_FP_EXTEND:
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::STRICT_FP_ROUND:
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::STRICT_FPOWI:
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP:
120 R = SoftenFloatRes_FFREXP(N);
121 break;
122 case ISD::STRICT_FREM:
123 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
124 case ISD::STRICT_FRINT:
125 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
126 case ISD::STRICT_FROUND:
127 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
128 case ISD::STRICT_FROUNDEVEN:
129 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
130 case ISD::STRICT_FSIN:
131 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
132 case ISD::STRICT_FSQRT:
133 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
134 case ISD::STRICT_FSUB:
135 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
136 case ISD::STRICT_FTRUNC:
137 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
138 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
139 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
140 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
141 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
142 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
143 case ISD::STRICT_SINT_TO_FP:
144 case ISD::STRICT_UINT_TO_FP:
145 case ISD::SINT_TO_FP:
146 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
147 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
148 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
149 case ISD::VECREDUCE_FADD:
150 case ISD::VECREDUCE_FMUL:
151 case ISD::VECREDUCE_FMIN:
152 case ISD::VECREDUCE_FMAX:
153 case ISD::VECREDUCE_FMAXIMUM:
154 case ISD::VECREDUCE_FMINIMUM:
155 R = SoftenFloatRes_VECREDUCE(N);
156 break;
157 case ISD::VECREDUCE_SEQ_FADD:
158 case ISD::VECREDUCE_SEQ_FMUL:
159 R = SoftenFloatRes_VECREDUCE_SEQ(N);
160 break;
161 }
162
163 // If R is null, the sub-method took care of registering the result.
164 if (R.getNode()) {
165 assert(R.getNode() != N);
166 SetSoftenedFloat(SDValue(N, ResNo), R);
167 }
168 }
169
170 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
171 bool IsStrict = N->isStrictFPOpcode();
172 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
173 unsigned Offset = IsStrict ? 1 : 0;
174 assert(N->getNumOperands() == (1 + Offset) &&
175 "Unexpected number of operands!");
176 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
177 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
178 TargetLowering::MakeLibCallOptions CallOptions;
179 EVT OpVT = N->getOperand(0 + Offset).getValueType();
180 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
181 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
182 CallOptions, SDLoc(N),
183 Chain);
184 if (IsStrict)
185 ReplaceValueWith(SDValue(N, 1), Tmp.second);
186 return Tmp.first;
187 }
188
189 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
190 bool IsStrict = N->isStrictFPOpcode();
191 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
192 unsigned Offset = IsStrict ? 1 : 0;
193 assert(N->getNumOperands() == (2 + Offset) &&
194 "Unexpected number of operands!");
195 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
196 GetSoftenedFloat(N->getOperand(1 + Offset)) };
197 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
198 TargetLowering::MakeLibCallOptions CallOptions;
199 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
200 N->getOperand(1 + Offset).getValueType() };
201 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
202 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
203 CallOptions, SDLoc(N),
204 Chain);
205 if (IsStrict)
206 ReplaceValueWith(SDValue(N, 1), Tmp.second);
207 return Tmp.first;
208 }
209
210 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
211 return BitConvertToInteger(N->getOperand(0));
212 }
213
214 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
215 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
216 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
217 GetSoftenedFloat(N->getOperand(0)));
218 }
219
220 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
221 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
222 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
223 GetSoftenedFloat(N->getOperand(0)));
224 return NewFence;
225 }
226
227 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
228 unsigned ResNo) {
229 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
230 return BitConvertToInteger(Op);
231 }
232
233 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
234 // Convert the inputs to integers, and build a new pair out of them.
235 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
236 TLI.getTypeToTransformTo(*DAG.getContext(),
237 N->getValueType(0)),
238 BitConvertToInteger(N->getOperand(0)),
239 BitConvertToInteger(N->getOperand(1)));
240 }
241
242 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
243 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
244 // In ppcf128, the high 64 bits are always first in memory regardless
245 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
246 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
247 // way. However, APInt's are serialized in an Endian-sensitive fashion,
248 // so on big-Endian targets, the two doubles are output in the wrong
249 // order. Fix this by manually flipping the order of the high 64 bits
250 // and the low 64 bits here.
251 if (DAG.getDataLayout().isBigEndian() &&
252 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
253 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
254 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
255 APInt Val(128, words);
256 return DAG.getConstant(Val, SDLoc(CN),
257 TLI.getTypeToTransformTo(*DAG.getContext(),
258 CN->getValueType(0)));
259 } else {
260 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
261 TLI.getTypeToTransformTo(*DAG.getContext(),
262 CN->getValueType(0)));
263 }
264 }
265
266 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
267 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
268 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
269 NewOp.getValueType().getVectorElementType(),
270 NewOp, N->getOperand(1));
271 }
272
273 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
274 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
275 unsigned Size = NVT.getSizeInBits();
276
277 // Mask = ~(1 << (Size-1))
278 APInt API = APInt::getAllOnes(Size);
279 API.clearBit(Size - 1);
280 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
281 SDValue Op = GetSoftenedFloat(N->getOperand(0));
282 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
283 }
284
285 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
286 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
287 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
288 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
289 RTLIB::FMIN_F32,
290 RTLIB::FMIN_F64,
291 RTLIB::FMIN_F80,
292 RTLIB::FMIN_F128,
293 RTLIB::FMIN_PPCF128));
294 }
295
296 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
297 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
298 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
299 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
300 RTLIB::FMAX_F32,
301 RTLIB::FMAX_F64,
302 RTLIB::FMAX_F80,
303 RTLIB::FMAX_F128,
304 RTLIB::FMAX_PPCF128));
305 }
306
307 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
308 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
309 RTLIB::ADD_F32,
310 RTLIB::ADD_F64,
311 RTLIB::ADD_F80,
312 RTLIB::ADD_F128,
313 RTLIB::ADD_PPCF128));
314 }
315
316 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
317 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
318 RTLIB::CBRT_F32,
319 RTLIB::CBRT_F64,
320 RTLIB::CBRT_F80,
321 RTLIB::CBRT_F128,
322 RTLIB::CBRT_PPCF128));
323 }
324
325 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
326 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
327 RTLIB::CEIL_F32,
328 RTLIB::CEIL_F64,
329 RTLIB::CEIL_F80,
330 RTLIB::CEIL_F128,
331 RTLIB::CEIL_PPCF128));
332 }
333
334 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
335 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
336 SDValue RHS = BitConvertToInteger(N->getOperand(1));
337 SDLoc dl(N);
338
339 EVT LVT = LHS.getValueType();
340 EVT RVT = RHS.getValueType();
341
342 unsigned LSize = LVT.getSizeInBits();
343 unsigned RSize = RVT.getSizeInBits();
344
345 // First get the sign bit of second operand.
346 SDValue SignBit = DAG.getNode(
347 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
348 DAG.getConstant(RSize - 1, dl,
349 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
350 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
351
352 // Shift right or sign-extend it if the two operands have different types.
353 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
354 if (SizeDiff > 0) {
355 SignBit =
356 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
357 DAG.getConstant(SizeDiff, dl,
358 TLI.getShiftAmountTy(SignBit.getValueType(),
359 DAG.getDataLayout())));
360 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
361 } else if (SizeDiff < 0) {
362 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
363 SignBit =
364 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
365 DAG.getConstant(-SizeDiff, dl,
366 TLI.getShiftAmountTy(SignBit.getValueType(),
367 DAG.getDataLayout())));
368 }
369
370 // Clear the sign bit of the first operand.
371 SDValue Mask = DAG.getNode(
372 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
373 DAG.getConstant(LSize - 1, dl,
374 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
375 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
376 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
377
378 // Or the value with the sign bit.
379 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
380 }
381
382 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
383 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
384 RTLIB::COS_F32,
385 RTLIB::COS_F64,
386 RTLIB::COS_F80,
387 RTLIB::COS_F128,
388 RTLIB::COS_PPCF128));
389 }
390
391 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
392 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
393 RTLIB::DIV_F32,
394 RTLIB::DIV_F64,
395 RTLIB::DIV_F80,
396 RTLIB::DIV_F128,
397 RTLIB::DIV_PPCF128));
398 }
399
400 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
401 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
402 RTLIB::EXP_F32,
403 RTLIB::EXP_F64,
404 RTLIB::EXP_F80,
405 RTLIB::EXP_F128,
406 RTLIB::EXP_PPCF128));
407 }
408
409 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
410 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
411 RTLIB::EXP2_F32,
412 RTLIB::EXP2_F64,
413 RTLIB::EXP2_F80,
414 RTLIB::EXP2_F128,
415 RTLIB::EXP2_PPCF128));
416 }
417
418 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
419 return SoftenFloatRes_Unary(
420 N,
421 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
422 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
423 }
424
425 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
426 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
427 RTLIB::FLOOR_F32,
428 RTLIB::FLOOR_F64,
429 RTLIB::FLOOR_F80,
430 RTLIB::FLOOR_F128,
431 RTLIB::FLOOR_PPCF128));
432 }
433
434 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
435 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
436 RTLIB::LOG_F32,
437 RTLIB::LOG_F64,
438 RTLIB::LOG_F80,
439 RTLIB::LOG_F128,
440 RTLIB::LOG_PPCF128));
441 }
442
443 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
444 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
445 RTLIB::LOG2_F32,
446 RTLIB::LOG2_F64,
447 RTLIB::LOG2_F80,
448 RTLIB::LOG2_F128,
449 RTLIB::LOG2_PPCF128));
450 }
451
452 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
453 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
454 RTLIB::LOG10_F32,
455 RTLIB::LOG10_F64,
456 RTLIB::LOG10_F80,
457 RTLIB::LOG10_F128,
458 RTLIB::LOG10_PPCF128));
459 }
460
461 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
462 bool IsStrict = N->isStrictFPOpcode();
463 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
464 unsigned Offset = IsStrict ? 1 : 0;
465 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
466 GetSoftenedFloat(N->getOperand(1 + Offset)),
467 GetSoftenedFloat(N->getOperand(2 + Offset)) };
468 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
469 TargetLowering::MakeLibCallOptions CallOptions;
470 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
471 N->getOperand(1 + Offset).getValueType(),
472 N->getOperand(2 + Offset).getValueType() };
473 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
474 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
475 GetFPLibCall(N->getValueType(0),
476 RTLIB::FMA_F32,
477 RTLIB::FMA_F64,
478 RTLIB::FMA_F80,
479 RTLIB::FMA_F128,
480 RTLIB::FMA_PPCF128),
481 NVT, Ops, CallOptions, SDLoc(N), Chain);
482 if (IsStrict)
483 ReplaceValueWith(SDValue(N, 1), Tmp.second);
484 return Tmp.first;
485 }
486
487 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
488 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
489 RTLIB::MUL_F32,
490 RTLIB::MUL_F64,
491 RTLIB::MUL_F80,
492 RTLIB::MUL_F128,
493 RTLIB::MUL_PPCF128));
494 }
495
496 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
497 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
498 RTLIB::NEARBYINT_F32,
499 RTLIB::NEARBYINT_F64,
500 RTLIB::NEARBYINT_F80,
501 RTLIB::NEARBYINT_F128,
502 RTLIB::NEARBYINT_PPCF128));
503 }
504
505 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
507 SDLoc dl(N);
508
509 // Expand Y = FNEG(X) -> Y = X ^ sign mask
510 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
511 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
512 DAG.getConstant(SignMask, dl, NVT));
513 }
514
515 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
516 bool IsStrict = N->isStrictFPOpcode();
517 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
518 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
519
520 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
521
522 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
523 Op = GetPromotedFloat(Op);
524 // If the promotion did the FP_EXTEND to the destination type for us,
525 // there's nothing left to do here.
526 if (Op.getValueType() == N->getValueType(0))
527 return BitConvertToInteger(Op);
528 }
529
530 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
531 // -> f32, so proceed in two stages. Also, it's entirely possible for both
532 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
533 // than FP16_TO_FP.
534 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
535 N->getValueType(0) != MVT::f32) {
536 if (IsStrict) {
537 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
538 { MVT::f32, MVT::Other }, { Chain, Op });
539 Chain = Op.getValue(1);
540 } else {
541 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
542 }
543 }
544
545 if (Op.getValueType() == MVT::bf16)
546 return SoftenFloatRes_BF16_TO_FP(N);
547
548 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
549 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
550 TargetLowering::MakeLibCallOptions CallOptions;
551 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
552 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
553 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
554 CallOptions, SDLoc(N),
555 Chain);
556 if (IsStrict)
557 ReplaceValueWith(SDValue(N, 1), Tmp.second);
558 return Tmp.first;
559 }
560
561 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
562 // nodes?
563 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
564 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
565 SDValue Op = N->getOperand(0);
566 TargetLowering::MakeLibCallOptions CallOptions;
567 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
568 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
569 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
570 CallOptions, SDLoc(N)).first;
571 if (N->getValueType(0) == MVT::f32)
572 return Res32;
573
574 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
575 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
576 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
577 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
578 }
579
580 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
581 // nodes?
582 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
583 assert(N->getValueType(0) == MVT::f32 &&
584 "Can only soften BF16_TO_FP with f32 result");
585 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
586 SDValue Op = N->getOperand(0);
587 SDLoc DL(N);
588 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
589 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
590 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
591 DAG.getShiftAmountConstant(16, NVT, DL));
592 return Res;
593 }
594
595 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
596 bool IsStrict = N->isStrictFPOpcode();
597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
598 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
599 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
600 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
601 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
602 TargetLowering::MakeLibCallOptions CallOptions;
603 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
604 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
605 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
606 CallOptions, SDLoc(N),
607 Chain);
608 if (IsStrict)
609 ReplaceValueWith(SDValue(N, 1), Tmp.second);
610 return Tmp.first;
611 }
612
613 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
614 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
615 RTLIB::POW_F32,
616 RTLIB::POW_F64,
617 RTLIB::POW_F80,
618 RTLIB::POW_F128,
619 RTLIB::POW_PPCF128));
620 }
621
622 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
623 bool IsStrict = N->isStrictFPOpcode();
624 unsigned Offset = IsStrict ? 1 : 0;
625 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
626 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
627 "Unsupported power type!");
628 bool IsPowI =
629 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
630
631 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
632 : RTLIB::getLDEXP(N->getValueType(0));
633 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
634 if (!TLI.getLibcallName(LC)) {
635 // Some targets don't have a powi libcall; use pow instead.
636 // FIXME: Implement this if some target needs it.
637 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
638 return DAG.getUNDEF(N->getValueType(0));
639 }
640
641 if (DAG.getLibInfo().getIntSize() !=
642 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
643 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
644 // would use the wrong type for the argument.
645 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
646 return DAG.getUNDEF(N->getValueType(0));
647 }
648
649 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
650 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
651 N->getOperand(1 + Offset) };
652 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
653 TargetLowering::MakeLibCallOptions CallOptions;
654 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
655 N->getOperand(1 + Offset).getValueType() };
656 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
657 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
658 CallOptions, SDLoc(N),
659 Chain);
660 if (IsStrict)
661 ReplaceValueWith(SDValue(N, 1), Tmp.second);
662 return Tmp.first;
663 }
664
665 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
666 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
667 EVT VT0 = N->getValueType(0);
668 EVT VT1 = N->getValueType(1);
669 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
670
671 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
672 // If the exponent does not match with sizeof(int) a libcall would use the
673 // wrong type for the argument.
674 // TODO: Should be able to handle mismatches.
675 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
676 return DAG.getUNDEF(N->getValueType(0));
677 }
678
679 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
680 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
681
682 SDLoc DL(N);
683
684 TargetLowering::MakeLibCallOptions CallOptions;
685 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
686 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
687
688 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
689 // but we only really need to handle the 0th one for softening anyway.
690 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
691
692 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
693 /*Chain=*/SDValue());
694 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
695 auto PtrInfo =
696 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
697
698 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
699
700 ReplaceValueWith(SDValue(N, 1), LoadExp);
701 return ReturnVal;
702 }
703
704 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
705 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
706 RTLIB::REM_F32,
707 RTLIB::REM_F64,
708 RTLIB::REM_F80,
709 RTLIB::REM_F128,
710 RTLIB::REM_PPCF128));
711 }
712
713 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
714 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
715 RTLIB::RINT_F32,
716 RTLIB::RINT_F64,
717 RTLIB::RINT_F80,
718 RTLIB::RINT_F128,
719 RTLIB::RINT_PPCF128));
720 }
721
722 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
723 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
724 RTLIB::ROUND_F32,
725 RTLIB::ROUND_F64,
726 RTLIB::ROUND_F80,
727 RTLIB::ROUND_F128,
728 RTLIB::ROUND_PPCF128));
729 }
730
731 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
732 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
733 RTLIB::ROUNDEVEN_F32,
734 RTLIB::ROUNDEVEN_F64,
735 RTLIB::ROUNDEVEN_F80,
736 RTLIB::ROUNDEVEN_F128,
737 RTLIB::ROUNDEVEN_PPCF128));
738 }
739
740 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
741 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
742 RTLIB::SIN_F32,
743 RTLIB::SIN_F64,
744 RTLIB::SIN_F80,
745 RTLIB::SIN_F128,
746 RTLIB::SIN_PPCF128));
747 }
748
749 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
750 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
751 RTLIB::SQRT_F32,
752 RTLIB::SQRT_F64,
753 RTLIB::SQRT_F80,
754 RTLIB::SQRT_F128,
755 RTLIB::SQRT_PPCF128));
756 }
757
758 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
759 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
760 RTLIB::SUB_F32,
761 RTLIB::SUB_F64,
762 RTLIB::SUB_F80,
763 RTLIB::SUB_F128,
764 RTLIB::SUB_PPCF128));
765 }
766
767 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
768 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
769 RTLIB::TRUNC_F32,
770 RTLIB::TRUNC_F64,
771 RTLIB::TRUNC_F80,
772 RTLIB::TRUNC_F128,
773 RTLIB::TRUNC_PPCF128));
774 }
775
776 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
777 LoadSDNode *L = cast<LoadSDNode>(N);
778 EVT VT = N->getValueType(0);
779 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
780 SDLoc dl(N);
781
782 auto MMOFlags =
783 L->getMemOperand()->getFlags() &
784 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
785 SDValue NewL;
786 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
787 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
788 L->getChain(), L->getBasePtr(), L->getOffset(),
789 L->getPointerInfo(), NVT, L->getOriginalAlign(),
790 MMOFlags, L->getAAInfo());
791 // Legalized the chain result - switch anything that used the old chain to
792 // use the new one.
793 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
794 return NewL;
795 }
796
797 // Do a non-extending load followed by FP_EXTEND.
798 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
799 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
800 L->getPointerInfo(), L->getMemoryVT(),
801 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
802 // Legalized the chain result - switch anything that used the old chain to
803 // use the new one.
804 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
805 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
806 return BitConvertToInteger(ExtendNode);
807 }
808
809 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
810 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
811 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
812 return DAG.getSelect(SDLoc(N),
813 LHS.getValueType(), N->getOperand(0), LHS, RHS);
814 }
815
816 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
817 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
818 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
819 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
820 LHS.getValueType(), N->getOperand(0),
821 N->getOperand(1), LHS, RHS, N->getOperand(4));
822 }
823
824 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
825 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
826 N->getValueType(0)));
827 }
828
829 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
830 SDValue Chain = N->getOperand(0); // Get the chain.
831 SDValue Ptr = N->getOperand(1); // Get the pointer.
832 EVT VT = N->getValueType(0);
833 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
834 SDLoc dl(N);
835
836 SDValue NewVAARG;
837 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
838 N->getConstantOperandVal(3));
839
840 // Legalized the chain result - switch anything that used the old chain to
841 // use the new one.
842 if (N != NewVAARG.getValue(1).getNode())
843 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
844 return NewVAARG;
845 }
846
847 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
848 bool IsStrict = N->isStrictFPOpcode();
849 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
850 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
851 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
852 EVT RVT = N->getValueType(0);
853 EVT NVT = EVT();
854 SDLoc dl(N);
855
856 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
857 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
858 // match. Look for an appropriate libcall.
859 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
860 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
861 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
862 NVT = (MVT::SimpleValueType)t;
863 // The source needs to big enough to hold the operand.
864 if (NVT.bitsGE(SVT))
865 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
866 }
867 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
868
869 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
870 // Sign/zero extend the argument if the libcall takes a larger type.
871 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
872 NVT, N->getOperand(IsStrict ? 1 : 0));
873 TargetLowering::MakeLibCallOptions CallOptions;
874 CallOptions.setSExt(Signed);
875 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
876 std::pair<SDValue, SDValue> Tmp =
877 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
878 Op, CallOptions, dl, Chain);
879
880 if (IsStrict)
881 ReplaceValueWith(SDValue(N, 1), Tmp.second);
882 return Tmp.first;
883 }
884
885 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
886 // Expand and soften recursively.
887 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
888 return SDValue();
889 }
890
891 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
892 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
893 return SDValue();
894 }
895
896 //===----------------------------------------------------------------------===//
897 // Convert Float Operand to Integer
898 //===----------------------------------------------------------------------===//
899
900 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
901 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
902 dbgs() << "\n");
903 SDValue Res = SDValue();
904
905 switch (N->getOpcode()) {
906 default:
907 #ifndef NDEBUG
908 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
909 N->dump(&DAG); dbgs() << "\n";
910 #endif
911 report_fatal_error("Do not know how to soften this operator's operand!");
912
913 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
914 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
915 case ISD::STRICT_FP_TO_FP16:
916 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
917 case ISD::FP_TO_BF16:
918 case ISD::STRICT_FP_ROUND:
919 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
920 case ISD::STRICT_FP_TO_SINT:
921 case ISD::STRICT_FP_TO_UINT:
922 case ISD::FP_TO_SINT:
923 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
924 case ISD::FP_TO_SINT_SAT:
925 case ISD::FP_TO_UINT_SAT:
926 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
927 case ISD::STRICT_LROUND:
928 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
929 case ISD::STRICT_LLROUND:
930 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
931 case ISD::STRICT_LRINT:
932 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
933 case ISD::STRICT_LLRINT:
934 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
935 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
936 case ISD::STRICT_FSETCC:
937 case ISD::STRICT_FSETCCS:
938 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
939 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
940 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
941 }
942
943 // If the result is null, the sub-method took care of registering results etc.
944 if (!Res.getNode()) return false;
945
946 // If the result is N, the sub-method updated N in place. Tell the legalizer
947 // core about this to re-analyze.
948 if (Res.getNode() == N)
949 return true;
950
951 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
952 "Invalid operand softening");
953
954 ReplaceValueWith(SDValue(N, 0), Res);
955 return false;
956 }
957
958 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
959 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
960
961 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
962 }
963
964 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
965 // We actually deal with the partially-softened FP_TO_FP16 node too, which
966 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
967 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
968 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
969 N->getOpcode() == ISD::FP_TO_BF16 ||
970 N->getOpcode() == ISD::STRICT_FP_ROUND);
971
972 bool IsStrict = N->isStrictFPOpcode();
973 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
974 EVT SVT = Op.getValueType();
975 EVT RVT = N->getValueType(0);
976 EVT FloatRVT = RVT;
977 if (N->getOpcode() == ISD::FP_TO_FP16 ||
978 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
979 FloatRVT = MVT::f16;
980 else if (N->getOpcode() == ISD::FP_TO_BF16)
981 FloatRVT = MVT::bf16;
982
983 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
984 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
985
986 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
987 Op = GetSoftenedFloat(Op);
988 TargetLowering::MakeLibCallOptions CallOptions;
989 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
990 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
991 CallOptions, SDLoc(N),
992 Chain);
993 if (IsStrict) {
994 ReplaceValueWith(SDValue(N, 1), Tmp.second);
995 ReplaceValueWith(SDValue(N, 0), Tmp.first);
996 return SDValue();
997 }
998 return Tmp.first;
999 }
1000
1001 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1002 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1003 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1004
1005 EVT VT = NewLHS.getValueType();
1006 NewLHS = GetSoftenedFloat(NewLHS);
1007 NewRHS = GetSoftenedFloat(NewRHS);
1008 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1009 N->getOperand(2), N->getOperand(3));
1010
1011 // If softenSetCCOperands returned a scalar, we need to compare the result
1012 // against zero to select between true and false values.
1013 if (!NewRHS.getNode()) {
1014 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1015 CCCode = ISD::SETNE;
1016 }
1017
1018 // Update N to have the operands specified.
1019 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1020 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1021 N->getOperand(4)),
1022 0);
1023 }
1024
1025 // Even if the result type is legal, no libcall may exactly match. (e.g. We
1026 // don't have FP-i8 conversions) This helper method looks for an appropriate
1027 // promoted libcall.
1028 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1029 bool Signed) {
1030 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1031 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1032 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1033 ++IntVT) {
1034 Promoted = (MVT::SimpleValueType)IntVT;
1035 // The type needs to big enough to hold the result.
1036 if (Promoted.bitsGE(RetVT))
1037 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1038 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1039 }
1040 return LC;
1041 }
1042
1043 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1044 bool IsStrict = N->isStrictFPOpcode();
1045 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1046 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1047
1048 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1049 EVT SVT = Op.getValueType();
1050 EVT RVT = N->getValueType(0);
1051 EVT NVT = EVT();
1052 SDLoc dl(N);
1053
1054 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1055 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1056 // match, eg. we don't have fp -> i8 conversions.
1057 // Look for an appropriate libcall.
1058 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1059 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1060 "Unsupported FP_TO_XINT!");
1061
1062 Op = GetSoftenedFloat(Op);
1063 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1064 TargetLowering::MakeLibCallOptions CallOptions;
1065 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1066 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1067 CallOptions, dl, Chain);
1068
1069 // Truncate the result if the libcall returns a larger type.
1070 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1071
1072 if (!IsStrict)
1073 return Res;
1074
1075 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1076 ReplaceValueWith(SDValue(N, 0), Res);
1077 return SDValue();
1078 }
1079
1080 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1081 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1082 return Res;
1083 }
1084
1085 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1086 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1087 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1088
1089 EVT VT = NewLHS.getValueType();
1090 NewLHS = GetSoftenedFloat(NewLHS);
1091 NewRHS = GetSoftenedFloat(NewRHS);
1092 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1093 N->getOperand(0), N->getOperand(1));
1094
1095 // If softenSetCCOperands returned a scalar, we need to compare the result
1096 // against zero to select between true and false values.
1097 if (!NewRHS.getNode()) {
1098 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1099 CCCode = ISD::SETNE;
1100 }
1101
1102 // Update N to have the operands specified.
1103 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1104 N->getOperand(2), N->getOperand(3),
1105 DAG.getCondCode(CCCode)),
1106 0);
1107 }
1108
1109 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1110 bool IsStrict = N->isStrictFPOpcode();
1111 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1112 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1113 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1114 ISD::CondCode CCCode =
1115 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1116
1117 EVT VT = Op0.getValueType();
1118 SDValue NewLHS = GetSoftenedFloat(Op0);
1119 SDValue NewRHS = GetSoftenedFloat(Op1);
1120 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1121 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1122
1123 // Update N to have the operands specified.
1124 if (NewRHS.getNode()) {
1125 if (IsStrict)
1126 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1127 NewRHS, DAG.getCondCode(CCCode));
1128 else
1129 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1130 DAG.getCondCode(CCCode)), 0);
1131 }
1132
1133 // Otherwise, softenSetCCOperands returned a scalar, use it.
1134 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1135 "Unexpected setcc expansion!");
1136
1137 if (IsStrict) {
1138 ReplaceValueWith(SDValue(N, 0), NewLHS);
1139 ReplaceValueWith(SDValue(N, 1), Chain);
1140 return SDValue();
1141 }
1142 return NewLHS;
1143 }
1144
1145 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1146 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1147 assert(OpNo == 1 && "Can only soften the stored value!");
1148 StoreSDNode *ST = cast<StoreSDNode>(N);
1149 SDValue Val = ST->getValue();
1150 SDLoc dl(N);
1151
1152 if (ST->isTruncatingStore())
1153 // Do an FP_ROUND followed by a non-truncating store.
1154 Val = BitConvertToInteger(
1155 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1156 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1157 else
1158 Val = GetSoftenedFloat(Val);
1159
1160 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1161 ST->getMemOperand());
1162 }
1163
1164 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1165 SDValue LHS = N->getOperand(0);
1166 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1167 SDLoc dl(N);
1168
1169 EVT LVT = LHS.getValueType();
1170 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1171 EVT RVT = RHS.getValueType();
1172
1173 unsigned LSize = LVT.getSizeInBits();
1174 unsigned RSize = RVT.getSizeInBits();
1175
1176 // Shift right or sign-extend it if the two operands have different types.
1177 int SizeDiff = RSize - LSize;
1178 if (SizeDiff > 0) {
1179 RHS =
1180 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1181 DAG.getConstant(SizeDiff, dl,
1182 TLI.getShiftAmountTy(RHS.getValueType(),
1183 DAG.getDataLayout())));
1184 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1185 } else if (SizeDiff < 0) {
1186 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1187 RHS =
1188 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1189 DAG.getConstant(-SizeDiff, dl,
1190 TLI.getShiftAmountTy(RHS.getValueType(),
1191 DAG.getDataLayout())));
1192 }
1193
1194 RHS = DAG.getBitcast(LVT, RHS);
1195 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1196 }
1197
1198 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1199 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1200 bool IsStrict = N->isStrictFPOpcode();
1201 unsigned Offset = IsStrict ? 1 : 0;
1202 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1203 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1204 TargetLowering::MakeLibCallOptions CallOptions;
1205 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1206 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1207 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1208 CallOptions, SDLoc(N),
1209 Chain);
1210 if (IsStrict) {
1211 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1212 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1213 return SDValue();
1214 }
1215
1216 return Tmp.first;
1217 }
1218
1219 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1220 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1221 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1222 RTLIB::LROUND_F32,
1223 RTLIB::LROUND_F64,
1224 RTLIB::LROUND_F80,
1225 RTLIB::LROUND_F128,
1226 RTLIB::LROUND_PPCF128));
1227 }
1228
1229 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1230 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1231 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1232 RTLIB::LLROUND_F32,
1233 RTLIB::LLROUND_F64,
1234 RTLIB::LLROUND_F80,
1235 RTLIB::LLROUND_F128,
1236 RTLIB::LLROUND_PPCF128));
1237 }
1238
1239 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1240 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1241 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1242 RTLIB::LRINT_F32,
1243 RTLIB::LRINT_F64,
1244 RTLIB::LRINT_F80,
1245 RTLIB::LRINT_F128,
1246 RTLIB::LRINT_PPCF128));
1247 }
1248
1249 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1250 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1251 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1252 RTLIB::LLRINT_F32,
1253 RTLIB::LLRINT_F64,
1254 RTLIB::LLRINT_F80,
1255 RTLIB::LLRINT_F128,
1256 RTLIB::LLRINT_PPCF128));
1257 }
1258
1259 //===----------------------------------------------------------------------===//
1260 // Float Result Expansion
1261 //===----------------------------------------------------------------------===//
1262
1263 /// ExpandFloatResult - This method is called when the specified result of the
1264 /// specified node is found to need expansion. At this point, the node may also
1265 /// have invalid operands or may have other results that need promotion, we just
1266 /// know that (at least) one result needs expansion.
1267 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1268 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1269 SDValue Lo, Hi;
1270 Lo = Hi = SDValue();
1271
1272 // See if the target wants to custom expand this node.
1273 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1274 return;
1275
1276 switch (N->getOpcode()) {
1277 default:
1278 #ifndef NDEBUG
1279 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1280 N->dump(&DAG); dbgs() << "\n";
1281 #endif
1282 report_fatal_error("Do not know how to expand the result of this "
1283 "operator!");
1284
1285 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1286 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1287 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1288
1289 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1290 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1291 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1292 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1293 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1294 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1295
1296 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1297 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1298 case ISD::STRICT_FMINNUM:
1299 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1300 case ISD::STRICT_FMAXNUM:
1301 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1302 case ISD::STRICT_FADD:
1303 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1304 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1305 case ISD::STRICT_FCEIL:
1306 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1307 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1308 case ISD::STRICT_FCOS:
1309 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1310 case ISD::STRICT_FDIV:
1311 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1312 case ISD::STRICT_FEXP:
1313 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1314 case ISD::STRICT_FEXP2:
1315 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1316 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1317 case ISD::STRICT_FFLOOR:
1318 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1319 case ISD::STRICT_FLOG:
1320 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1321 case ISD::STRICT_FLOG2:
1322 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1323 case ISD::STRICT_FLOG10:
1324 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1325 case ISD::STRICT_FMA:
1326 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1327 case ISD::STRICT_FMUL:
1328 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1329 case ISD::STRICT_FNEARBYINT:
1330 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1331 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1332 case ISD::STRICT_FP_EXTEND:
1333 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1334 case ISD::STRICT_FPOW:
1335 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1336 case ISD::STRICT_FPOWI:
1337 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1338 case ISD::FLDEXP:
1339 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1340 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1341 case ISD::STRICT_FRINT:
1342 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1343 case ISD::STRICT_FROUND:
1344 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1345 case ISD::STRICT_FROUNDEVEN:
1346 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1347 case ISD::STRICT_FSIN:
1348 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1349 case ISD::STRICT_FSQRT:
1350 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1351 case ISD::STRICT_FSUB:
1352 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1353 case ISD::STRICT_FTRUNC:
1354 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1355 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1356 case ISD::STRICT_SINT_TO_FP:
1357 case ISD::STRICT_UINT_TO_FP:
1358 case ISD::SINT_TO_FP:
1359 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1360 case ISD::STRICT_FREM:
1361 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1362 }
1363
1364 // If Lo/Hi is null, the sub-method took care of registering results etc.
1365 if (Lo.getNode())
1366 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1367 }
1368
1369 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1370 SDValue &Hi) {
1371 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1372 assert(NVT.getSizeInBits() == 64 &&
1373 "Do not know how to expand this float constant!");
1374 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1375 SDLoc dl(N);
1376 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1377 APInt(64, C.getRawData()[1])),
1378 dl, NVT);
1379 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1380 APInt(64, C.getRawData()[0])),
1381 dl, NVT);
1382 }
1383
1384 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1385 SDValue &Lo, SDValue &Hi) {
1386 bool IsStrict = N->isStrictFPOpcode();
1387 unsigned Offset = IsStrict ? 1 : 0;
1388 SDValue Op = N->getOperand(0 + Offset);
1389 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1390 TargetLowering::MakeLibCallOptions CallOptions;
1391 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1392 Op, CallOptions, SDLoc(N),
1393 Chain);
1394 if (IsStrict)
1395 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1396 GetPairElements(Tmp.first, Lo, Hi);
1397 }
1398
1399 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1400 SDValue &Lo, SDValue &Hi) {
1401 bool IsStrict = N->isStrictFPOpcode();
1402 unsigned Offset = IsStrict ? 1 : 0;
1403 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1404 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1405 TargetLowering::MakeLibCallOptions CallOptions;
1406 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1407 Ops, CallOptions, SDLoc(N),
1408 Chain);
1409 if (IsStrict)
1410 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1411 GetPairElements(Tmp.first, Lo, Hi);
1412 }
1413
1414 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1415 SDValue &Hi) {
1416 assert(N->getValueType(0) == MVT::ppcf128 &&
1417 "Logic only correct for ppcf128!");
1418 SDLoc dl(N);
1419 SDValue Tmp;
1420 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1421 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1422 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1423 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1424 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1425 ISD::SETEQ);
1426 }
1427
1428 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1429 SDValue &Hi) {
1430 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1431 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1432 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1433 RTLIB::FMIN_PPCF128), Lo, Hi);
1434 }
1435
1436 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1437 SDValue &Hi) {
1438 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1439 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1440 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1441 RTLIB::FMAX_PPCF128), Lo, Hi);
1442 }
1443
1444 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1445 SDValue &Hi) {
1446 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1447 RTLIB::ADD_F32, RTLIB::ADD_F64,
1448 RTLIB::ADD_F80, RTLIB::ADD_F128,
1449 RTLIB::ADD_PPCF128), Lo, Hi);
1450 }
1451
1452 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1453 SDValue &Hi) {
1454 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1455 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1456 RTLIB::CBRT_F128,
1457 RTLIB::CBRT_PPCF128), Lo, Hi);
1458 }
1459
1460 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1461 SDValue &Lo, SDValue &Hi) {
1462 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1463 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1464 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1465 RTLIB::CEIL_PPCF128), Lo, Hi);
1466 }
1467
1468 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1469 SDValue &Lo, SDValue &Hi) {
1470 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1471 RTLIB::COPYSIGN_F32,
1472 RTLIB::COPYSIGN_F64,
1473 RTLIB::COPYSIGN_F80,
1474 RTLIB::COPYSIGN_F128,
1475 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1476 }
1477
1478 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1479 SDValue &Lo, SDValue &Hi) {
1480 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1481 RTLIB::COS_F32, RTLIB::COS_F64,
1482 RTLIB::COS_F80, RTLIB::COS_F128,
1483 RTLIB::COS_PPCF128), Lo, Hi);
1484 }
1485
1486 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1487 SDValue &Hi) {
1488 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1489 RTLIB::DIV_F32,
1490 RTLIB::DIV_F64,
1491 RTLIB::DIV_F80,
1492 RTLIB::DIV_F128,
1493 RTLIB::DIV_PPCF128), Lo, Hi);
1494 }
1495
1496 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1497 SDValue &Lo, SDValue &Hi) {
1498 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1499 RTLIB::EXP_F32, RTLIB::EXP_F64,
1500 RTLIB::EXP_F80, RTLIB::EXP_F128,
1501 RTLIB::EXP_PPCF128), Lo, Hi);
1502 }
1503
1504 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1505 SDValue &Lo, SDValue &Hi) {
1506 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1507 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1508 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1509 RTLIB::EXP2_PPCF128), Lo, Hi);
1510 }
1511
1512 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1513 SDValue &Hi) {
1514 ExpandFloatRes_Unary(N,
1515 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1516 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1517 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1518 Lo, Hi);
1519 }
1520
1521 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1522 SDValue &Lo, SDValue &Hi) {
1523 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1524 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1525 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1526 RTLIB::FLOOR_PPCF128), Lo, Hi);
1527 }
1528
1529 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1530 SDValue &Lo, SDValue &Hi) {
1531 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1532 RTLIB::LOG_F32, RTLIB::LOG_F64,
1533 RTLIB::LOG_F80, RTLIB::LOG_F128,
1534 RTLIB::LOG_PPCF128), Lo, Hi);
1535 }
1536
1537 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1538 SDValue &Lo, SDValue &Hi) {
1539 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1540 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1541 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1542 RTLIB::LOG2_PPCF128), Lo, Hi);
1543 }
1544
1545 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1546 SDValue &Lo, SDValue &Hi) {
1547 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1548 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1549 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1550 RTLIB::LOG10_PPCF128), Lo, Hi);
1551 }
1552
1553 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1554 SDValue &Hi) {
1555 bool IsStrict = N->isStrictFPOpcode();
1556 unsigned Offset = IsStrict ? 1 : 0;
1557 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1558 N->getOperand(2 + Offset) };
1559 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1560 TargetLowering::MakeLibCallOptions CallOptions;
1561 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1562 RTLIB::FMA_F32,
1563 RTLIB::FMA_F64,
1564 RTLIB::FMA_F80,
1565 RTLIB::FMA_F128,
1566 RTLIB::FMA_PPCF128),
1567 N->getValueType(0), Ops, CallOptions,
1568 SDLoc(N), Chain);
1569 if (IsStrict)
1570 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1571 GetPairElements(Tmp.first, Lo, Hi);
1572 }
1573
1574 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1575 SDValue &Hi) {
1576 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1577 RTLIB::MUL_F32,
1578 RTLIB::MUL_F64,
1579 RTLIB::MUL_F80,
1580 RTLIB::MUL_F128,
1581 RTLIB::MUL_PPCF128), Lo, Hi);
1582 }
1583
1584 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1585 SDValue &Lo, SDValue &Hi) {
1586 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1587 RTLIB::NEARBYINT_F32,
1588 RTLIB::NEARBYINT_F64,
1589 RTLIB::NEARBYINT_F80,
1590 RTLIB::NEARBYINT_F128,
1591 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1592 }
1593
1594 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1595 SDValue &Hi) {
1596 SDLoc dl(N);
1597 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1598 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1599 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1600 }
1601
1602 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1603 SDValue &Hi) {
1604 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1605 SDLoc dl(N);
1606 bool IsStrict = N->isStrictFPOpcode();
1607
1608 SDValue Chain;
1609 if (IsStrict) {
1610 // If the expanded type is the same as the input type, just bypass the node.
1611 if (NVT == N->getOperand(1).getValueType()) {
1612 Hi = N->getOperand(1);
1613 Chain = N->getOperand(0);
1614 } else {
1615 // Other we need to extend.
1616 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1617 { N->getOperand(0), N->getOperand(1) });
1618 Chain = Hi.getValue(1);
1619 }
1620 } else {
1621 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1622 }
1623
1624 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1625 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1626
1627 if (IsStrict)
1628 ReplaceValueWith(SDValue(N, 1), Chain);
1629 }
1630
1631 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1632 SDValue &Lo, SDValue &Hi) {
1633 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1634 RTLIB::POW_F32, RTLIB::POW_F64,
1635 RTLIB::POW_F80, RTLIB::POW_F128,
1636 RTLIB::POW_PPCF128), Lo, Hi);
1637 }
1638
1639 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1640 SDValue &Lo, SDValue &Hi) {
1641 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1642 }
1643
1644 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1645 SDValue &Hi) {
1646 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1647 }
1648
1649 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1650 SDValue &Lo, SDValue &Hi) {
1651 assert(N->getValueType(0) == MVT::ppcf128 &&
1652 "Logic only correct for ppcf128!");
1653
1654 SDLoc dl(N);
1655 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1656 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1657 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1658 }
1659
1660 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1663 RTLIB::REM_F32, RTLIB::REM_F64,
1664 RTLIB::REM_F80, RTLIB::REM_F128,
1665 RTLIB::REM_PPCF128), Lo, Hi);
1666 }
1667
1668 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1669 SDValue &Lo, SDValue &Hi) {
1670 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1671 RTLIB::RINT_F32, RTLIB::RINT_F64,
1672 RTLIB::RINT_F80, RTLIB::RINT_F128,
1673 RTLIB::RINT_PPCF128), Lo, Hi);
1674 }
1675
1676 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1677 SDValue &Lo, SDValue &Hi) {
1678 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1679 RTLIB::ROUND_F32,
1680 RTLIB::ROUND_F64,
1681 RTLIB::ROUND_F80,
1682 RTLIB::ROUND_F128,
1683 RTLIB::ROUND_PPCF128), Lo, Hi);
1684 }
1685
1686 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1687 SDValue &Lo, SDValue &Hi) {
1688 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1689 RTLIB::ROUNDEVEN_F32,
1690 RTLIB::ROUNDEVEN_F64,
1691 RTLIB::ROUNDEVEN_F80,
1692 RTLIB::ROUNDEVEN_F128,
1693 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1694 }
1695
1696 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1697 SDValue &Lo, SDValue &Hi) {
1698 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1699 RTLIB::SIN_F32, RTLIB::SIN_F64,
1700 RTLIB::SIN_F80, RTLIB::SIN_F128,
1701 RTLIB::SIN_PPCF128), Lo, Hi);
1702 }
1703
1704 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1705 SDValue &Lo, SDValue &Hi) {
1706 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1707 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1708 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1709 RTLIB::SQRT_PPCF128), Lo, Hi);
1710 }
1711
1712 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1713 SDValue &Hi) {
1714 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1715 RTLIB::SUB_F32,
1716 RTLIB::SUB_F64,
1717 RTLIB::SUB_F80,
1718 RTLIB::SUB_F128,
1719 RTLIB::SUB_PPCF128), Lo, Hi);
1720 }
1721
1722 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1723 SDValue &Lo, SDValue &Hi) {
1724 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1725 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1726 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1727 RTLIB::TRUNC_PPCF128), Lo, Hi);
1728 }
1729
1730 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 if (ISD::isNormalLoad(N)) {
1733 ExpandRes_NormalLoad(N, Lo, Hi);
1734 return;
1735 }
1736
1737 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1738 LoadSDNode *LD = cast<LoadSDNode>(N);
1739 SDValue Chain = LD->getChain();
1740 SDValue Ptr = LD->getBasePtr();
1741 SDLoc dl(N);
1742
1743 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1744 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1745 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1746
1747 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1748 LD->getMemoryVT(), LD->getMemOperand());
1749
1750 // Remember the chain.
1751 Chain = Hi.getValue(1);
1752
1753 // The low part is zero.
1754 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1755 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1756
1757 // Modified the chain - switch anything that used the old chain to use the
1758 // new one.
1759 ReplaceValueWith(SDValue(LD, 1), Chain);
1760 }
1761
1762 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1763 SDValue &Hi) {
1764 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1765 EVT VT = N->getValueType(0);
1766 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1767 bool Strict = N->isStrictFPOpcode();
1768 SDValue Src = N->getOperand(Strict ? 1 : 0);
1769 EVT SrcVT = Src.getValueType();
1770 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1771 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1772 SDLoc dl(N);
1773 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1774
1775 // TODO: Any other flags to propagate?
1776 SDNodeFlags Flags;
1777 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1778
1779 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1780 // When promoting partial word types to i32 we must honor the signedness,
1781 // though.
1782 if (SrcVT.bitsLE(MVT::i32)) {
1783 // The integer can be represented exactly in an f64.
1784 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1785 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1786 if (Strict) {
1787 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1788 {Chain, Src}, Flags);
1789 Chain = Hi.getValue(1);
1790 } else
1791 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1792 } else {
1793 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1794 if (SrcVT.bitsLE(MVT::i64)) {
1795 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1796 MVT::i64, Src);
1797 LC = RTLIB::SINTTOFP_I64_PPCF128;
1798 } else if (SrcVT.bitsLE(MVT::i128)) {
1799 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1800 LC = RTLIB::SINTTOFP_I128_PPCF128;
1801 }
1802 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1803
1804 TargetLowering::MakeLibCallOptions CallOptions;
1805 CallOptions.setSExt(true);
1806 std::pair<SDValue, SDValue> Tmp =
1807 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1808 if (Strict)
1809 Chain = Tmp.second;
1810 GetPairElements(Tmp.first, Lo, Hi);
1811 }
1812
1813 // No need to complement for unsigned 32-bit integers
1814 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1815 if (Strict)
1816 ReplaceValueWith(SDValue(N, 1), Chain);
1817
1818 return;
1819 }
1820
1821 // Unsigned - fix up the SINT_TO_FP value just calculated.
1822 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1823 // keep semantics correctness if the integer is not exactly representable
1824 // here. See ExpandLegalINT_TO_FP.
1825 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1826 SrcVT = Src.getValueType();
1827
1828 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1829 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1830 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1831 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1832 ArrayRef<uint64_t> Parts;
1833
1834 switch (SrcVT.getSimpleVT().SimpleTy) {
1835 default:
1836 llvm_unreachable("Unsupported UINT_TO_FP!");
1837 case MVT::i32:
1838 Parts = TwoE32;
1839 break;
1840 case MVT::i64:
1841 Parts = TwoE64;
1842 break;
1843 case MVT::i128:
1844 Parts = TwoE128;
1845 break;
1846 }
1847
1848 // TODO: Are there other fast-math-flags to propagate to this FADD?
1849 SDValue NewLo = DAG.getConstantFP(
1850 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1851 if (Strict) {
1852 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1853 {Chain, Hi, NewLo}, Flags);
1854 Chain = Lo.getValue(1);
1855 ReplaceValueWith(SDValue(N, 1), Chain);
1856 } else
1857 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1858 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1859 Lo, Hi, ISD::SETLT);
1860 GetPairElements(Lo, Lo, Hi);
1861 }
1862
1863
1864 //===----------------------------------------------------------------------===//
1865 // Float Operand Expansion
1866 //===----------------------------------------------------------------------===//
1867
1868 /// ExpandFloatOperand - This method is called when the specified operand of the
1869 /// specified node is found to need expansion. At this point, all of the result
1870 /// types of the node are known to be legal, but other operands of the node may
1871 /// need promotion or expansion as well as the specified one.
1872 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1873 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1874 SDValue Res = SDValue();
1875
1876 // See if the target wants to custom expand this node.
1877 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1878 return false;
1879
1880 switch (N->getOpcode()) {
1881 default:
1882 #ifndef NDEBUG
1883 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1884 N->dump(&DAG); dbgs() << "\n";
1885 #endif
1886 report_fatal_error("Do not know how to expand this operator's operand!");
1887
1888 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1889 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1890 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1891
1892 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1893 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1894 case ISD::STRICT_FP_ROUND:
1895 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1896 case ISD::STRICT_FP_TO_SINT:
1897 case ISD::STRICT_FP_TO_UINT:
1898 case ISD::FP_TO_SINT:
1899 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1900 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1901 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1902 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1903 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1904 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1905 case ISD::STRICT_FSETCC:
1906 case ISD::STRICT_FSETCCS:
1907 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1908 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1909 OpNo); break;
1910 }
1911
1912 // If the result is null, the sub-method took care of registering results etc.
1913 if (!Res.getNode()) return false;
1914
1915 // If the result is N, the sub-method updated N in place. Tell the legalizer
1916 // core about this.
1917 if (Res.getNode() == N)
1918 return true;
1919
1920 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1921 "Invalid operand expansion");
1922
1923 ReplaceValueWith(SDValue(N, 0), Res);
1924 return false;
1925 }
1926
1927 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1928 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1929 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1930 SDValue &NewRHS,
1931 ISD::CondCode &CCCode,
1932 const SDLoc &dl, SDValue &Chain,
1933 bool IsSignaling) {
1934 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1935 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1936 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1937
1938 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1939
1940 // FIXME: This generated code sucks. We want to generate
1941 // FCMPU crN, hi1, hi2
1942 // BNE crN, L:
1943 // FCMPU crN, lo1, lo2
1944 // The following can be improved, but not that much.
1945 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1946 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1947 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1948 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1949 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1950 RHSLo, CCCode, OutputChain, IsSignaling);
1951 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1952 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1953 Tmp1 =
1954 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1955 ISD::SETUNE, OutputChain, IsSignaling);
1956 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1957 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1958 RHSHi, CCCode, OutputChain, IsSignaling);
1959 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1960 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1961 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1962 NewRHS = SDValue(); // LHS is the result, not a compare.
1963 Chain = OutputChain;
1964 }
1965
1966 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1967 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1968 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1969 SDValue Chain;
1970 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1971
1972 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1973 // against zero to select between true and false values.
1974 if (!NewRHS.getNode()) {
1975 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1976 CCCode = ISD::SETNE;
1977 }
1978
1979 // Update N to have the operands specified.
1980 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1981 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1982 N->getOperand(4)), 0);
1983 }
1984
1985 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1986 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1987 "Logic only correct for ppcf128!");
1988 SDValue Lo, Hi;
1989 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1990 // The ppcf128 value is providing only the sign; take it from the
1991 // higher-order double (which must have the larger magnitude).
1992 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1993 N->getValueType(0), N->getOperand(0), Hi);
1994 }
1995
1996 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1997 bool IsStrict = N->isStrictFPOpcode();
1998 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1999 "Logic only correct for ppcf128!");
2000 SDValue Lo, Hi;
2001 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2002
2003 if (!IsStrict)
2004 // Round it the rest of the way (e.g. to f32) if needed.
2005 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2006 N->getValueType(0), Hi, N->getOperand(1));
2007
2008 // Eliminate the node if the input float type is the same as the output float
2009 // type.
2010 if (Hi.getValueType() == N->getValueType(0)) {
2011 // Connect the output chain to the input chain, unlinking the node.
2012 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2013 ReplaceValueWith(SDValue(N, 0), Hi);
2014 return SDValue();
2015 }
2016
2017 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2018 {N->getValueType(0), MVT::Other},
2019 {N->getOperand(0), Hi, N->getOperand(2)});
2020 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2021 ReplaceValueWith(SDValue(N, 0), Expansion);
2022 return SDValue();
2023 }
2024
2025 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2026 EVT RVT = N->getValueType(0);
2027 SDLoc dl(N);
2028
2029 bool IsStrict = N->isStrictFPOpcode();
2030 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2031 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2032 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2033 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2034
2035 EVT NVT;
2036 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2037 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2038 "Unsupported FP_TO_XINT!");
2039 TargetLowering::MakeLibCallOptions CallOptions;
2040 std::pair<SDValue, SDValue> Tmp =
2041 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2042 if (!IsStrict)
2043 return Tmp.first;
2044
2045 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2046 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2047 return SDValue();
2048 }
2049
2050 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2051 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2052 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2053 SDValue Chain;
2054 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2055
2056 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2057 // against zero to select between true and false values.
2058 if (!NewRHS.getNode()) {
2059 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2060 CCCode = ISD::SETNE;
2061 }
2062
2063 // Update N to have the operands specified.
2064 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2065 N->getOperand(2), N->getOperand(3),
2066 DAG.getCondCode(CCCode)), 0);
2067 }
2068
2069 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2070 bool IsStrict = N->isStrictFPOpcode();
2071 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2072 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2073 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2074 ISD::CondCode CCCode =
2075 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2076 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2077 N->getOpcode() == ISD::STRICT_FSETCCS);
2078
2079 // FloatExpandSetCCOperands always returned a scalar.
2080 assert(!NewRHS.getNode() && "Expect to return scalar");
2081 assert(NewLHS.getValueType() == N->getValueType(0) &&
2082 "Unexpected setcc expansion!");
2083 if (Chain) {
2084 ReplaceValueWith(SDValue(N, 0), NewLHS);
2085 ReplaceValueWith(SDValue(N, 1), Chain);
2086 return SDValue();
2087 }
2088 return NewLHS;
2089 }
2090
2091 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2092 if (ISD::isNormalStore(N))
2093 return ExpandOp_NormalStore(N, OpNo);
2094
2095 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2096 assert(OpNo == 1 && "Can only expand the stored value so far");
2097 StoreSDNode *ST = cast<StoreSDNode>(N);
2098
2099 SDValue Chain = ST->getChain();
2100 SDValue Ptr = ST->getBasePtr();
2101
2102 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2103 ST->getValue().getValueType());
2104 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2105 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2106 (void)NVT;
2107
2108 SDValue Lo, Hi;
2109 GetExpandedOp(ST->getValue(), Lo, Hi);
2110
2111 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2112 ST->getMemoryVT(), ST->getMemOperand());
2113 }
2114
2115 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2116 EVT RVT = N->getValueType(0);
2117 EVT RetVT = N->getOperand(0).getValueType();
2118 TargetLowering::MakeLibCallOptions CallOptions;
2119 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2120 RTLIB::LROUND_F32,
2121 RTLIB::LROUND_F64,
2122 RTLIB::LROUND_F80,
2123 RTLIB::LROUND_F128,
2124 RTLIB::LROUND_PPCF128),
2125 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2126 }
2127
2128 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2129 EVT RVT = N->getValueType(0);
2130 EVT RetVT = N->getOperand(0).getValueType();
2131 TargetLowering::MakeLibCallOptions CallOptions;
2132 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2133 RTLIB::LLROUND_F32,
2134 RTLIB::LLROUND_F64,
2135 RTLIB::LLROUND_F80,
2136 RTLIB::LLROUND_F128,
2137 RTLIB::LLROUND_PPCF128),
2138 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2139 }
2140
2141 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2142 EVT RVT = N->getValueType(0);
2143 EVT RetVT = N->getOperand(0).getValueType();
2144 TargetLowering::MakeLibCallOptions CallOptions;
2145 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2146 RTLIB::LRINT_F32,
2147 RTLIB::LRINT_F64,
2148 RTLIB::LRINT_F80,
2149 RTLIB::LRINT_F128,
2150 RTLIB::LRINT_PPCF128),
2151 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2152 }
2153
2154 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2155 EVT RVT = N->getValueType(0);
2156 EVT RetVT = N->getOperand(0).getValueType();
2157 TargetLowering::MakeLibCallOptions CallOptions;
2158 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2159 RTLIB::LLRINT_F32,
2160 RTLIB::LLRINT_F64,
2161 RTLIB::LLRINT_F80,
2162 RTLIB::LLRINT_F128,
2163 RTLIB::LLRINT_PPCF128),
2164 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2165 }
2166
2167 //===----------------------------------------------------------------------===//
2168 // Float Operand Promotion
2169 //===----------------------------------------------------------------------===//
2170 //
2171
2172 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2173 if (OpVT == MVT::f16) {
2174 return ISD::FP16_TO_FP;
2175 } else if (RetVT == MVT::f16) {
2176 return ISD::FP_TO_FP16;
2177 } else if (OpVT == MVT::bf16) {
2178 return ISD::BF16_TO_FP;
2179 } else if (RetVT == MVT::bf16) {
2180 return ISD::FP_TO_BF16;
2181 }
2182
2183 report_fatal_error("Attempt at an invalid promotion-related conversion");
2184 }
2185
2186 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2187 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2188 dbgs() << "\n");
2189 SDValue R = SDValue();
2190
2191 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2192 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2193 return false;
2194 }
2195
2196 // Nodes that use a promotion-requiring floating point operand, but doesn't
2197 // produce a promotion-requiring floating point result, need to be legalized
2198 // to use the promoted float operand. Nodes that produce at least one
2199 // promotion-requiring floating point result have their operands legalized as
2200 // a part of PromoteFloatResult.
2201 // clang-format off
2202 switch (N->getOpcode()) {
2203 default:
2204 #ifndef NDEBUG
2205 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2206 N->dump(&DAG); dbgs() << "\n";
2207 #endif
2208 report_fatal_error("Do not know how to promote this operator's operand!");
2209
2210 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2211 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2212 case ISD::FP_TO_SINT:
2213 case ISD::FP_TO_UINT:
2214 case ISD::LRINT:
2215 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2216 case ISD::FP_TO_SINT_SAT:
2217 case ISD::FP_TO_UINT_SAT:
2218 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2219 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2220 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2221 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2222 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2223 }
2224 // clang-format on
2225
2226 if (R.getNode())
2227 ReplaceValueWith(SDValue(N, 0), R);
2228 return false;
2229 }
2230
2231 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2232 SDValue Op = N->getOperand(0);
2233 EVT OpVT = Op->getValueType(0);
2234
2235 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2236 EVT PromotedVT = Promoted->getValueType(0);
2237
2238 // Convert the promoted float value to the desired IVT.
2239 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2240 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2241 IVT, Promoted);
2242 // The final result type might not be an scalar so we need a bitcast. The
2243 // bitcast will be further legalized if needed.
2244 return DAG.getBitcast(N->getValueType(0), Convert);
2245 }
2246
2247 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2248 // PromoteFloatRes_FCOPYSIGN.
2249 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2250 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2251 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2252
2253 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2254 N->getOperand(0), Op1);
2255 }
2256
2257 // Convert the promoted float value to the desired integer type
2258 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2259 SDValue Op = GetPromotedFloat(N->getOperand(0));
2260 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2261 }
2262
2263 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2264 unsigned OpNo) {
2265 SDValue Op = GetPromotedFloat(N->getOperand(0));
2266 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2267 N->getOperand(1));
2268 }
2269
2270 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2271 SDValue Op = GetPromotedFloat(N->getOperand(0));
2272 EVT VT = N->getValueType(0);
2273
2274 // Desired VT is same as promoted type. Use promoted float directly.
2275 if (VT == Op->getValueType(0))
2276 return Op;
2277
2278 // Else, extend the promoted float value to the desired VT.
2279 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2280 }
2281
2282 // Promote the float operands used for comparison. The true- and false-
2283 // operands have the same type as the result and are promoted, if needed, by
2284 // PromoteFloatRes_SELECT_CC
2285 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2286 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2287 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2288
2289 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2290 LHS, RHS, N->getOperand(2), N->getOperand(3),
2291 N->getOperand(4));
2292 }
2293
2294 // Construct a SETCC that compares the promoted values and sets the conditional
2295 // code.
2296 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2297 EVT VT = N->getValueType(0);
2298 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2299 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2300 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2301
2302 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2303
2304 }
2305
2306 // Lower the promoted Float down to the integer value of same size and construct
2307 // a STORE of the integer value.
2308 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2309 StoreSDNode *ST = cast<StoreSDNode>(N);
2310 SDValue Val = ST->getValue();
2311 SDLoc DL(N);
2312
2313 SDValue Promoted = GetPromotedFloat(Val);
2314 EVT VT = ST->getOperand(1).getValueType();
2315 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2316
2317 SDValue NewVal;
2318 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2319 IVT, Promoted);
2320
2321 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2322 ST->getMemOperand());
2323 }
2324
2325 //===----------------------------------------------------------------------===//
2326 // Float Result Promotion
2327 //===----------------------------------------------------------------------===//
2328
2329 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2330 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2331 dbgs() << "\n");
2332 SDValue R = SDValue();
2333
2334 // See if the target wants to custom expand this node.
2335 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2336 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2337 return;
2338 }
2339
2340 switch (N->getOpcode()) {
2341 // These opcodes cannot appear if promotion of FP16 is done in the backend
2342 // instead of Clang
2343 case ISD::FP16_TO_FP:
2344 case ISD::FP_TO_FP16:
2345 default:
2346 #ifndef NDEBUG
2347 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2348 N->dump(&DAG); dbgs() << "\n";
2349 #endif
2350 report_fatal_error("Do not know how to promote this operator's result!");
2351
2352 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2353 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2354 case ISD::EXTRACT_VECTOR_ELT:
2355 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2356 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2357
2358 // Unary FP Operations
2359 case ISD::FABS:
2360 case ISD::FCBRT:
2361 case ISD::FCEIL:
2362 case ISD::FCOS:
2363 case ISD::FEXP:
2364 case ISD::FEXP2:
2365 case ISD::FEXP10:
2366 case ISD::FFLOOR:
2367 case ISD::FLOG:
2368 case ISD::FLOG2:
2369 case ISD::FLOG10:
2370 case ISD::FNEARBYINT:
2371 case ISD::FNEG:
2372 case ISD::FRINT:
2373 case ISD::FROUND:
2374 case ISD::FROUNDEVEN:
2375 case ISD::FSIN:
2376 case ISD::FSQRT:
2377 case ISD::FTRUNC:
2378 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2379
2380 // Binary FP Operations
2381 case ISD::FADD:
2382 case ISD::FDIV:
2383 case ISD::FMAXIMUM:
2384 case ISD::FMINIMUM:
2385 case ISD::FMAXNUM:
2386 case ISD::FMINNUM:
2387 case ISD::FMUL:
2388 case ISD::FPOW:
2389 case ISD::FREM:
2390 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2391
2392 case ISD::FMA: // FMA is same as FMAD
2393 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2394
2395 case ISD::FPOWI:
2396 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2397 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2398
2399 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2400 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2401 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2402 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2403
2404 case ISD::SINT_TO_FP:
2405 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2406 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2407 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2408 case ISD::VECREDUCE_FADD:
2409 case ISD::VECREDUCE_FMUL:
2410 case ISD::VECREDUCE_FMIN:
2411 case ISD::VECREDUCE_FMAX:
2412 case ISD::VECREDUCE_FMAXIMUM:
2413 case ISD::VECREDUCE_FMINIMUM:
2414 R = PromoteFloatRes_VECREDUCE(N);
2415 break;
2416 case ISD::VECREDUCE_SEQ_FADD:
2417 case ISD::VECREDUCE_SEQ_FMUL:
2418 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2419 break;
2420 }
2421
2422 if (R.getNode())
2423 SetPromotedFloat(SDValue(N, ResNo), R);
2424 }
2425
2426 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2427 // At this point, it is not possible to determine if the bitcast value is
2428 // eventually stored to memory or promoted to f32 or promoted to a floating
2429 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2430 // STORE promotion handlers.
2431 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2432 EVT VT = N->getValueType(0);
2433 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2434 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2435 // bitcast will be legalized further if necessary.
2436 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2437 N->getOperand(0).getValueType().getSizeInBits());
2438 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2439 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2440 }
2441
2442 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2443 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2444 EVT VT = N->getValueType(0);
2445 SDLoc DL(N);
2446
2447 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2448 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2449 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2450 IVT);
2451
2452 // Convert the Constant to the desired FP type
2453 // FIXME We might be able to do the conversion during compilation and get rid
2454 // of it from the object code
2455 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2456 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2457 }
2458
2459 // If the Index operand is a constant, try to redirect the extract operation to
2460 // the correct legalized vector. If not, bit-convert the input vector to
2461 // equivalent integer vector. Extract the element as an (bit-cast) integer
2462 // value and convert it to the promoted type.
2463 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2464 SDLoc DL(N);
2465
2466 // If the index is constant, try to extract the value from the legalized
2467 // vector type.
2468 if (isa<ConstantSDNode>(N->getOperand(1))) {
2469 SDValue Vec = N->getOperand(0);
2470 SDValue Idx = N->getOperand(1);
2471 EVT VecVT = Vec->getValueType(0);
2472 EVT EltVT = VecVT.getVectorElementType();
2473
2474 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2475
2476 switch (getTypeAction(VecVT)) {
2477 default: break;
2478 case TargetLowering::TypeScalarizeVector: {
2479 SDValue Res = GetScalarizedVector(N->getOperand(0));
2480 ReplaceValueWith(SDValue(N, 0), Res);
2481 return SDValue();
2482 }
2483 case TargetLowering::TypeWidenVector: {
2484 Vec = GetWidenedVector(Vec);
2485 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2486 ReplaceValueWith(SDValue(N, 0), Res);
2487 return SDValue();
2488 }
2489 case TargetLowering::TypeSplitVector: {
2490 SDValue Lo, Hi;
2491 GetSplitVector(Vec, Lo, Hi);
2492
2493 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2494 SDValue Res;
2495 if (IdxVal < LoElts)
2496 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2497 else
2498 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2499 DAG.getConstant(IdxVal - LoElts, DL,
2500 Idx.getValueType()));
2501 ReplaceValueWith(SDValue(N, 0), Res);
2502 return SDValue();
2503 }
2504
2505 }
2506 }
2507
2508 // Bit-convert the input vector to the equivalent integer vector
2509 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2510 EVT IVT = NewOp.getValueType().getVectorElementType();
2511
2512 // Extract the element as an (bit-cast) integer value
2513 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2514 NewOp, N->getOperand(1));
2515
2516 // Convert the element to the desired FP type
2517 EVT VT = N->getValueType(0);
2518 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2519 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2520 }
2521
2522 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2523 // needs promotion, so does the argument X. Note that Y, if needed, will be
2524 // handled during operand promotion.
2525 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2526 EVT VT = N->getValueType(0);
2527 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2528 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2529
2530 SDValue Op1 = N->getOperand(1);
2531
2532 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2533 }
2534
2535 // Unary operation where the result and the operand have PromoteFloat type
2536 // action. Construct a new SDNode with the promoted float value of the old
2537 // operand.
2538 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2539 EVT VT = N->getValueType(0);
2540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2541 SDValue Op = GetPromotedFloat(N->getOperand(0));
2542
2543 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2544 }
2545
2546 // Binary operations where the result and both operands have PromoteFloat type
2547 // action. Construct a new SDNode with the promoted float values of the old
2548 // operands.
2549 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2550 EVT VT = N->getValueType(0);
2551 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2552 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2553 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2554 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2555 }
2556
2557 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2558 EVT VT = N->getValueType(0);
2559 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2560 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2561 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2562 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2563
2564 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2565 }
2566
2567 // Promote the Float (first) operand and retain the Integer (second) operand
2568 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2569 EVT VT = N->getValueType(0);
2570 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2571 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2572 SDValue Op1 = N->getOperand(1);
2573
2574 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2575 }
2576
2577 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2578 EVT VT = N->getValueType(0);
2579 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2580 SDValue Op = GetPromotedFloat(N->getOperand(0));
2581 SDValue Res =
2582 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2583
2584 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2585 return Res;
2586 }
2587
2588 // Explicit operation to reduce precision. Reduce the value to half precision
2589 // and promote it back to the legal type.
2590 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2591 SDLoc DL(N);
2592
2593 SDValue Op = N->getOperand(0);
2594 EVT VT = N->getValueType(0);
2595 EVT OpVT = Op->getValueType(0);
2596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2597 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2598
2599 // Round promoted float to desired precision
2600 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2601 // Promote it back to the legal output type
2602 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2603 }
2604
2605 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2606 LoadSDNode *L = cast<LoadSDNode>(N);
2607 EVT VT = N->getValueType(0);
2608
2609 // Load the value as an integer value with the same number of bits.
2610 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2611 SDValue newL = DAG.getLoad(
2612 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2613 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2614 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2615 // Legalize the chain result by replacing uses of the old value chain with the
2616 // new one
2617 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2618
2619 // Convert the integer value to the desired FP type
2620 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2621 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2622 }
2623
2624 // Construct a new SELECT node with the promoted true- and false- values.
2625 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2626 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2627 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2628
2629 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2630 N->getOperand(0), TrueVal, FalseVal);
2631 }
2632
2633 // Construct a new SELECT_CC node with the promoted true- and false- values.
2634 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2635 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2636 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2637 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2638
2639 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2640 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2641 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2642 }
2643
2644 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2645 // float type.
2646 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2647 SDLoc DL(N);
2648 EVT VT = N->getValueType(0);
2649 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2650 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2651 // Round the value to the desired precision (that of the source type).
2652 return DAG.getNode(
2653 ISD::FP_EXTEND, DL, NVT,
2654 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2655 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2656 }
2657
2658 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2659 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2660 N->getValueType(0)));
2661 }
2662
2663 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2664 // Expand and promote recursively.
2665 // TODO: This is non-optimal, but dealing with the concurrently happening
2666 // vector-legalization is non-trivial. We could do something similar to
2667 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2668 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2669 return SDValue();
2670 }
2671
2672 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2673 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2674 return SDValue();
2675 }
2676
2677 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2678 EVT VT = N->getValueType(0);
2679
2680 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2681 SDLoc SL(N);
2682
2683 SDValue CastVal = BitConvertToInteger(AM->getVal());
2684 EVT CastVT = CastVal.getValueType();
2685
2686 SDValue NewAtomic
2687 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2688 DAG.getVTList(CastVT, MVT::Other),
2689 { AM->getChain(), AM->getBasePtr(), CastVal },
2690 AM->getMemOperand());
2691
2692 SDValue Result = NewAtomic;
2693
2694 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2695 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2696 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2697 NewAtomic);
2698 }
2699
2700 // Legalize the chain result by replacing uses of the old value chain with the
2701 // new one
2702 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2703
2704 return Result;
2705
2706 }
2707
2708 //===----------------------------------------------------------------------===//
2709 // Half Result Soft Promotion
2710 //===----------------------------------------------------------------------===//
2711
2712 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2713 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2714 N->dump(&DAG); dbgs() << "\n");
2715 SDValue R = SDValue();
2716
2717 // See if the target wants to custom expand this node.
2718 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2719 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2720 return;
2721 }
2722
2723 switch (N->getOpcode()) {
2724 default:
2725 #ifndef NDEBUG
2726 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2727 N->dump(&DAG); dbgs() << "\n";
2728 #endif
2729 report_fatal_error("Do not know how to soft promote this operator's "
2730 "result!");
2731
2732 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2733 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2734 case ISD::EXTRACT_VECTOR_ELT:
2735 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2736 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2737 case ISD::STRICT_FP_ROUND:
2738 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2739
2740 // Unary FP Operations
2741 case ISD::FABS:
2742 case ISD::FCBRT:
2743 case ISD::FCEIL:
2744 case ISD::FCOS:
2745 case ISD::FEXP:
2746 case ISD::FEXP2:
2747 case ISD::FEXP10:
2748 case ISD::FFLOOR:
2749 case ISD::FLOG:
2750 case ISD::FLOG2:
2751 case ISD::FLOG10:
2752 case ISD::FNEARBYINT:
2753 case ISD::FNEG:
2754 case ISD::FREEZE:
2755 case ISD::FRINT:
2756 case ISD::FROUND:
2757 case ISD::FROUNDEVEN:
2758 case ISD::FSIN:
2759 case ISD::FSQRT:
2760 case ISD::FTRUNC:
2761 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2762
2763 // Binary FP Operations
2764 case ISD::FADD:
2765 case ISD::FDIV:
2766 case ISD::FMAXIMUM:
2767 case ISD::FMINIMUM:
2768 case ISD::FMAXNUM:
2769 case ISD::FMINNUM:
2770 case ISD::FMUL:
2771 case ISD::FPOW:
2772 case ISD::FREM:
2773 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2774
2775 case ISD::FMA: // FMA is same as FMAD
2776 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2777
2778 case ISD::FPOWI:
2779 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2780
2781 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2782 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2783 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2784 case ISD::SINT_TO_FP:
2785 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2786 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2787 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2788 case ISD::VECREDUCE_FADD:
2789 case ISD::VECREDUCE_FMUL:
2790 case ISD::VECREDUCE_FMIN:
2791 case ISD::VECREDUCE_FMAX:
2792 case ISD::VECREDUCE_FMAXIMUM:
2793 case ISD::VECREDUCE_FMINIMUM:
2794 R = SoftPromoteHalfRes_VECREDUCE(N);
2795 break;
2796 case ISD::VECREDUCE_SEQ_FADD:
2797 case ISD::VECREDUCE_SEQ_FMUL:
2798 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2799 break;
2800 }
2801
2802 if (R.getNode())
2803 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2804 }
2805
2806 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2807 return BitConvertToInteger(N->getOperand(0));
2808 }
2809
2810 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2811 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2812
2813 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2814 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2815 MVT::i16);
2816 }
2817
2818 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2819 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2820 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2821 NewOp.getValueType().getVectorElementType(), NewOp,
2822 N->getOperand(1));
2823 }
2824
2825 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2826 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2827 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2828 SDLoc dl(N);
2829
2830 EVT LVT = LHS.getValueType();
2831 EVT RVT = RHS.getValueType();
2832
2833 unsigned LSize = LVT.getSizeInBits();
2834 unsigned RSize = RVT.getSizeInBits();
2835
2836 // First get the sign bit of second operand.
2837 SDValue SignBit = DAG.getNode(
2838 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2839 DAG.getConstant(RSize - 1, dl,
2840 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2841 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2842
2843 // Shift right or sign-extend it if the two operands have different types.
2844 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2845 if (SizeDiff > 0) {
2846 SignBit =
2847 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2848 DAG.getConstant(SizeDiff, dl,
2849 TLI.getShiftAmountTy(SignBit.getValueType(),
2850 DAG.getDataLayout())));
2851 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2852 } else if (SizeDiff < 0) {
2853 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2854 SignBit =
2855 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2856 DAG.getConstant(-SizeDiff, dl,
2857 TLI.getShiftAmountTy(SignBit.getValueType(),
2858 DAG.getDataLayout())));
2859 }
2860
2861 // Clear the sign bit of the first operand.
2862 SDValue Mask = DAG.getNode(
2863 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2864 DAG.getConstant(LSize - 1, dl,
2865 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2866 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2867 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2868
2869 // Or the value with the sign bit.
2870 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2871 }
2872
2873 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2874 EVT OVT = N->getValueType(0);
2875 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2876 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2877 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2878 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2879 SDLoc dl(N);
2880
2881 // Promote to the larger FP type.
2882 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2883 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2884 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2885 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2886
2887 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2888
2889 // Convert back to FP16 as an integer.
2890 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2891 }
2892
2893 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2894 EVT OVT = N->getValueType(0);
2895 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2896 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2897 SDValue Op1 = N->getOperand(1);
2898 SDLoc dl(N);
2899
2900 // Promote to the larger FP type.
2901 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2902
2903 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2904
2905 // Convert back to FP16 as an integer.
2906 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2907 }
2908
2909 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2910 EVT RVT = N->getValueType(0);
2911 EVT SVT = N->getOperand(0).getValueType();
2912
2913 if (N->isStrictFPOpcode()) {
2914 assert(RVT == MVT::f16);
2915 SDValue Res =
2916 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2917 {N->getOperand(0), N->getOperand(1)});
2918 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2919 return Res;
2920 }
2921
2922 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2923 N->getOperand(0));
2924 }
2925
2926 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2927 LoadSDNode *L = cast<LoadSDNode>(N);
2928
2929 // Load the value as an integer value with the same number of bits.
2930 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2931 SDValue NewL =
2932 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2933 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2934 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2935 L->getMemOperand()->getFlags(), L->getAAInfo());
2936 // Legalize the chain result by replacing uses of the old value chain with the
2937 // new one
2938 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2939 return NewL;
2940 }
2941
2942 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2943 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2944 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2945 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2946 Op2);
2947 }
2948
2949 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2950 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2951 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2952 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2953 N->getOperand(0), N->getOperand(1), Op2, Op3,
2954 N->getOperand(4));
2955 }
2956
2957 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2958 EVT OVT = N->getValueType(0);
2959 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2960 SDLoc dl(N);
2961
2962 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2963
2964 // Round the value to the softened type.
2965 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2966 }
2967
2968 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2969 return DAG.getUNDEF(MVT::i16);
2970 }
2971
2972 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2973 EVT OVT = N->getValueType(0);
2974 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2975 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2976 SDLoc dl(N);
2977
2978 // Promote to the larger FP type.
2979 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2980
2981 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2982
2983 // Convert back to FP16 as an integer.
2984 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2985 }
2986
2987 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2988 EVT OVT = N->getValueType(0);
2989 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2990 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2991 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2992 SDLoc dl(N);
2993
2994 // Promote to the larger FP type.
2995 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2996 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2997 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2998
2999 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3000
3001 // Convert back to FP16 as an integer.
3002 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3003 }
3004
3005 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3006 // Expand and soften recursively.
3007 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3008 return SDValue();
3009 }
3010
3011 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3012 // Expand and soften.
3013 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3014 return SDValue();
3015 }
3016
3017 //===----------------------------------------------------------------------===//
3018 // Half Operand Soft Promotion
3019 //===----------------------------------------------------------------------===//
3020
3021 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3022 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3023 N->dump(&DAG); dbgs() << "\n");
3024 SDValue Res = SDValue();
3025
3026 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3027 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3028 return false;
3029 }
3030
3031 // Nodes that use a promotion-requiring floating point operand, but doesn't
3032 // produce a soft promotion-requiring floating point result, need to be
3033 // legalized to use the soft promoted float operand. Nodes that produce at
3034 // least one soft promotion-requiring floating point result have their
3035 // operands legalized as a part of PromoteFloatResult.
3036 switch (N->getOpcode()) {
3037 default:
3038 #ifndef NDEBUG
3039 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3040 N->dump(&DAG); dbgs() << "\n";
3041 #endif
3042 report_fatal_error("Do not know how to soft promote this operator's "
3043 "operand!");
3044
3045 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3046 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3047 case ISD::FP_TO_SINT:
3048 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3049 case ISD::FP_TO_SINT_SAT:
3050 case ISD::FP_TO_UINT_SAT:
3051 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3052 case ISD::STRICT_FP_EXTEND:
3053 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3054 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3055 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3056 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3057 case ISD::STACKMAP:
3058 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3059 break;
3060 case ISD::PATCHPOINT:
3061 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3062 break;
3063 }
3064
3065 if (!Res.getNode())
3066 return false;
3067
3068 assert(Res.getNode() != N && "Expected a new node!");
3069
3070 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3071 "Invalid operand expansion");
3072
3073 ReplaceValueWith(SDValue(N, 0), Res);
3074 return false;
3075 }
3076
3077 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3078 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3079
3080 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3081 }
3082
3083 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3084 unsigned OpNo) {
3085 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3086 SDValue Op1 = N->getOperand(1);
3087 EVT RVT = Op1.getValueType();
3088 SDLoc dl(N);
3089
3090 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3091
3092 Op1 = GetSoftPromotedHalf(Op1);
3093 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3094
3095 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3096 Op1);
3097 }
3098
3099 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3100 EVT RVT = N->getValueType(0);
3101 bool IsStrict = N->isStrictFPOpcode();
3102 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3103 EVT SVT = Op.getValueType();
3104 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3105
3106 if (IsStrict) {
3107 assert(SVT == MVT::f16);
3108 SDValue Res =
3109 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
3110 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3111 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3112 ReplaceValueWith(SDValue(N, 0), Res);
3113 return SDValue();
3114 }
3115
3116 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3117 }
3118
3119 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3120 EVT RVT = N->getValueType(0);
3121 SDValue Op = N->getOperand(0);
3122 EVT SVT = Op.getValueType();
3123 SDLoc dl(N);
3124
3125 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3126
3127 Op = GetSoftPromotedHalf(Op);
3128
3129 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3130
3131 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3132 }
3133
3134 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3135 EVT RVT = N->getValueType(0);
3136 SDValue Op = N->getOperand(0);
3137 EVT SVT = Op.getValueType();
3138 SDLoc dl(N);
3139
3140 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3141
3142 Op = GetSoftPromotedHalf(Op);
3143
3144 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3145
3146 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3147 N->getOperand(1));
3148 }
3149
3150 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3151 unsigned OpNo) {
3152 assert(OpNo == 0 && "Can only soften the comparison values");
3153 SDValue Op0 = N->getOperand(0);
3154 SDValue Op1 = N->getOperand(1);
3155 SDLoc dl(N);
3156
3157 EVT SVT = Op0.getValueType();
3158 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3159
3160 Op0 = GetSoftPromotedHalf(Op0);
3161 Op1 = GetSoftPromotedHalf(Op1);
3162
3163 // Promote to the larger FP type.
3164 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3165 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3166 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3167
3168 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3169 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3170 }
3171
3172 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3173 SDValue Op0 = N->getOperand(0);
3174 SDValue Op1 = N->getOperand(1);
3175 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3176 SDLoc dl(N);
3177
3178 EVT SVT = Op0.getValueType();
3179 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3180
3181 Op0 = GetSoftPromotedHalf(Op0);
3182 Op1 = GetSoftPromotedHalf(Op1);
3183
3184 // Promote to the larger FP type.
3185 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3186 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3187 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3188
3189 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3190 }
3191
3192 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3193 assert(OpNo == 1 && "Can only soften the stored value!");
3194 StoreSDNode *ST = cast<StoreSDNode>(N);
3195 SDValue Val = ST->getValue();
3196 SDLoc dl(N);
3197
3198 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3199 SDValue Promoted = GetSoftPromotedHalf(Val);
3200 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3201 ST->getMemOperand());
3202 }
3203
3204 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3205 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3206 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3207 SDValue Op = N->getOperand(OpNo);
3208 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3209 SDValue NewNode =
3210 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3211
3212 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3213 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3214
3215 return SDValue(); // Signal that we replaced the node ourselves.
3216 }
3217
3218 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3219 unsigned OpNo) {
3220 assert(OpNo >= 7);
3221 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3222 SDValue Op = N->getOperand(OpNo);
3223 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3224 SDValue NewNode =
3225 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3226
3227 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3228 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3229
3230 return SDValue(); // Signal that we replaced the node ourselves.
3231 }
LLVM monorepo