?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[ORC] Add std::tuple support to SimplePackedSerialization.
[llvm-project.git] / llvm / lib / CodeGen / SelectionDAG / LegalizeFloatTypes.cpp
blob664a3c15027747e9701b4b159c3bb6a8d83ff3fa
1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58 #ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61 #endif
62 llvm_unreachable("Do not know how to soften the result of this operator!");
63
64 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
65 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
66 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
67 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
68 case ISD::EXTRACT_VECTOR_ELT:
69 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
70 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
71 case ISD::STRICT_FMINNUM:
72 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
73 case ISD::STRICT_FMAXNUM:
74 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
75 case ISD::STRICT_FADD:
76 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
77 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
78 case ISD::STRICT_FCEIL:
79 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
80 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
81 case ISD::STRICT_FCOS:
82 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
83 case ISD::STRICT_FDIV:
84 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
85 case ISD::STRICT_FEXP:
86 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
87 case ISD::STRICT_FEXP2:
88 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
89 case ISD::STRICT_FFLOOR:
90 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
91 case ISD::STRICT_FLOG:
92 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
93 case ISD::STRICT_FLOG2:
94 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
95 case ISD::STRICT_FLOG10:
96 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
97 case ISD::STRICT_FMA:
98 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
99 case ISD::STRICT_FMUL:
100 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
101 case ISD::STRICT_FNEARBYINT:
102 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
103 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
104 case ISD::STRICT_FP_EXTEND:
105 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
106 case ISD::STRICT_FP_ROUND:
107 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
108 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
109 case ISD::STRICT_FPOW:
110 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
111 case ISD::STRICT_FPOWI:
112 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
113 case ISD::STRICT_FREM:
114 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
115 case ISD::STRICT_FRINT:
116 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
117 case ISD::STRICT_FROUND:
118 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
119 case ISD::STRICT_FROUNDEVEN:
120 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
121 case ISD::STRICT_FSIN:
122 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
123 case ISD::STRICT_FSQRT:
124 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
125 case ISD::STRICT_FSUB:
126 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
127 case ISD::STRICT_FTRUNC:
128 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
129 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
130 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
131 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
132 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
133 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
134 case ISD::STRICT_SINT_TO_FP:
135 case ISD::STRICT_UINT_TO_FP:
136 case ISD::SINT_TO_FP:
137 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
138 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
139 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
140 case ISD::VECREDUCE_FADD:
141 case ISD::VECREDUCE_FMUL:
142 case ISD::VECREDUCE_FMIN:
143 case ISD::VECREDUCE_FMAX:
144 R = SoftenFloatRes_VECREDUCE(N);
145 break;
146 case ISD::VECREDUCE_SEQ_FADD:
147 case ISD::VECREDUCE_SEQ_FMUL:
148 R = SoftenFloatRes_VECREDUCE_SEQ(N);
149 break;
150 }
151
152 // If R is null, the sub-method took care of registering the result.
153 if (R.getNode()) {
154 assert(R.getNode() != N);
155 SetSoftenedFloat(SDValue(N, ResNo), R);
156 }
157 }
158
159 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
160 bool IsStrict = N->isStrictFPOpcode();
161 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
162 unsigned Offset = IsStrict ? 1 : 0;
163 assert(N->getNumOperands() == (1 + Offset) &&
164 "Unexpected number of operands!");
165 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
166 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
167 TargetLowering::MakeLibCallOptions CallOptions;
168 EVT OpVT = N->getOperand(0 + Offset).getValueType();
169 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
170 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
171 CallOptions, SDLoc(N),
172 Chain);
173 if (IsStrict)
174 ReplaceValueWith(SDValue(N, 1), Tmp.second);
175 return Tmp.first;
176 }
177
178 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
179 bool IsStrict = N->isStrictFPOpcode();
180 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
181 unsigned Offset = IsStrict ? 1 : 0;
182 assert(N->getNumOperands() == (2 + Offset) &&
183 "Unexpected number of operands!");
184 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
185 GetSoftenedFloat(N->getOperand(1 + Offset)) };
186 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
187 TargetLowering::MakeLibCallOptions CallOptions;
188 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
189 N->getOperand(1 + Offset).getValueType() };
190 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
192 CallOptions, SDLoc(N),
193 Chain);
194 if (IsStrict)
195 ReplaceValueWith(SDValue(N, 1), Tmp.second);
196 return Tmp.first;
197 }
198
199 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
200 return BitConvertToInteger(N->getOperand(0));
201 }
202
203 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
204 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
205 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
206 GetSoftenedFloat(N->getOperand(0)));
207 }
208
209 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
210 unsigned ResNo) {
211 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
212 return BitConvertToInteger(Op);
213 }
214
215 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
216 // Convert the inputs to integers, and build a new pair out of them.
217 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
218 TLI.getTypeToTransformTo(*DAG.getContext(),
219 N->getValueType(0)),
220 BitConvertToInteger(N->getOperand(0)),
221 BitConvertToInteger(N->getOperand(1)));
222 }
223
224 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
225 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
226 // In ppcf128, the high 64 bits are always first in memory regardless
227 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
228 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
229 // way. However, APInt's are serialized in an Endian-sensitive fashion,
230 // so on big-Endian targets, the two doubles are output in the wrong
231 // order. Fix this by manually flipping the order of the high 64 bits
232 // and the low 64 bits here.
233 if (DAG.getDataLayout().isBigEndian() &&
234 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
235 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
236 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
237 APInt Val(128, words);
238 return DAG.getConstant(Val, SDLoc(CN),
239 TLI.getTypeToTransformTo(*DAG.getContext(),
240 CN->getValueType(0)));
241 } else {
242 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
243 TLI.getTypeToTransformTo(*DAG.getContext(),
244 CN->getValueType(0)));
245 }
246 }
247
248 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
249 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
250 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
251 NewOp.getValueType().getVectorElementType(),
252 NewOp, N->getOperand(1));
253 }
254
255 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
257 unsigned Size = NVT.getSizeInBits();
258
259 // Mask = ~(1 << (Size-1))
260 APInt API = APInt::getAllOnesValue(Size);
261 API.clearBit(Size - 1);
262 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
263 SDValue Op = GetSoftenedFloat(N->getOperand(0));
264 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
265 }
266
267 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
268 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
269 RTLIB::FMIN_F32,
270 RTLIB::FMIN_F64,
271 RTLIB::FMIN_F80,
272 RTLIB::FMIN_F128,
273 RTLIB::FMIN_PPCF128));
274 }
275
276 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
277 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
278 RTLIB::FMAX_F32,
279 RTLIB::FMAX_F64,
280 RTLIB::FMAX_F80,
281 RTLIB::FMAX_F128,
282 RTLIB::FMAX_PPCF128));
283 }
284
285 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
286 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
287 RTLIB::ADD_F32,
288 RTLIB::ADD_F64,
289 RTLIB::ADD_F80,
290 RTLIB::ADD_F128,
291 RTLIB::ADD_PPCF128));
292 }
293
294 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
295 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
296 RTLIB::CBRT_F32,
297 RTLIB::CBRT_F64,
298 RTLIB::CBRT_F80,
299 RTLIB::CBRT_F128,
300 RTLIB::CBRT_PPCF128));
301 }
302
303 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
304 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
305 RTLIB::CEIL_F32,
306 RTLIB::CEIL_F64,
307 RTLIB::CEIL_F80,
308 RTLIB::CEIL_F128,
309 RTLIB::CEIL_PPCF128));
310 }
311
312 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
313 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
314 SDValue RHS = BitConvertToInteger(N->getOperand(1));
315 SDLoc dl(N);
316
317 EVT LVT = LHS.getValueType();
318 EVT RVT = RHS.getValueType();
319
320 unsigned LSize = LVT.getSizeInBits();
321 unsigned RSize = RVT.getSizeInBits();
322
323 // First get the sign bit of second operand.
324 SDValue SignBit = DAG.getNode(
325 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
326 DAG.getConstant(RSize - 1, dl,
327 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
328 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
329
330 // Shift right or sign-extend it if the two operands have different types.
331 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
332 if (SizeDiff > 0) {
333 SignBit =
334 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
335 DAG.getConstant(SizeDiff, dl,
336 TLI.getShiftAmountTy(SignBit.getValueType(),
337 DAG.getDataLayout())));
338 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
339 } else if (SizeDiff < 0) {
340 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
341 SignBit =
342 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
343 DAG.getConstant(-SizeDiff, dl,
344 TLI.getShiftAmountTy(SignBit.getValueType(),
345 DAG.getDataLayout())));
346 }
347
348 // Clear the sign bit of the first operand.
349 SDValue Mask = DAG.getNode(
350 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
351 DAG.getConstant(LSize - 1, dl,
352 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
353 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
354 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
355
356 // Or the value with the sign bit.
357 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
358 }
359
360 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
361 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
362 RTLIB::COS_F32,
363 RTLIB::COS_F64,
364 RTLIB::COS_F80,
365 RTLIB::COS_F128,
366 RTLIB::COS_PPCF128));
367 }
368
369 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
370 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
371 RTLIB::DIV_F32,
372 RTLIB::DIV_F64,
373 RTLIB::DIV_F80,
374 RTLIB::DIV_F128,
375 RTLIB::DIV_PPCF128));
376 }
377
378 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
379 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
380 RTLIB::EXP_F32,
381 RTLIB::EXP_F64,
382 RTLIB::EXP_F80,
383 RTLIB::EXP_F128,
384 RTLIB::EXP_PPCF128));
385 }
386
387 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
388 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
389 RTLIB::EXP2_F32,
390 RTLIB::EXP2_F64,
391 RTLIB::EXP2_F80,
392 RTLIB::EXP2_F128,
393 RTLIB::EXP2_PPCF128));
394 }
395
396 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
397 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
398 RTLIB::FLOOR_F32,
399 RTLIB::FLOOR_F64,
400 RTLIB::FLOOR_F80,
401 RTLIB::FLOOR_F128,
402 RTLIB::FLOOR_PPCF128));
403 }
404
405 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
406 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
407 RTLIB::LOG_F32,
408 RTLIB::LOG_F64,
409 RTLIB::LOG_F80,
410 RTLIB::LOG_F128,
411 RTLIB::LOG_PPCF128));
412 }
413
414 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
415 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
416 RTLIB::LOG2_F32,
417 RTLIB::LOG2_F64,
418 RTLIB::LOG2_F80,
419 RTLIB::LOG2_F128,
420 RTLIB::LOG2_PPCF128));
421 }
422
423 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
424 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
425 RTLIB::LOG10_F32,
426 RTLIB::LOG10_F64,
427 RTLIB::LOG10_F80,
428 RTLIB::LOG10_F128,
429 RTLIB::LOG10_PPCF128));
430 }
431
432 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
433 bool IsStrict = N->isStrictFPOpcode();
434 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
435 unsigned Offset = IsStrict ? 1 : 0;
436 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
437 GetSoftenedFloat(N->getOperand(1 + Offset)),
438 GetSoftenedFloat(N->getOperand(2 + Offset)) };
439 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
440 TargetLowering::MakeLibCallOptions CallOptions;
441 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
442 N->getOperand(1 + Offset).getValueType(),
443 N->getOperand(2 + Offset).getValueType() };
444 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
445 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
446 GetFPLibCall(N->getValueType(0),
447 RTLIB::FMA_F32,
448 RTLIB::FMA_F64,
449 RTLIB::FMA_F80,
450 RTLIB::FMA_F128,
451 RTLIB::FMA_PPCF128),
452 NVT, Ops, CallOptions, SDLoc(N), Chain);
453 if (IsStrict)
454 ReplaceValueWith(SDValue(N, 1), Tmp.second);
455 return Tmp.first;
456 }
457
458 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
459 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
460 RTLIB::MUL_F32,
461 RTLIB::MUL_F64,
462 RTLIB::MUL_F80,
463 RTLIB::MUL_F128,
464 RTLIB::MUL_PPCF128));
465 }
466
467 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
468 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
469 RTLIB::NEARBYINT_F32,
470 RTLIB::NEARBYINT_F64,
471 RTLIB::NEARBYINT_F80,
472 RTLIB::NEARBYINT_F128,
473 RTLIB::NEARBYINT_PPCF128));
474 }
475
476 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
477 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
478 SDLoc dl(N);
479
480 // Expand Y = FNEG(X) -> Y = X ^ sign mask
481 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
482 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
483 DAG.getConstant(SignMask, dl, NVT));
484 }
485
486 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
487 bool IsStrict = N->isStrictFPOpcode();
488 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
489 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
490
491 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
492
493 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
494 Op = GetPromotedFloat(Op);
495 // If the promotion did the FP_EXTEND to the destination type for us,
496 // there's nothing left to do here.
497 if (Op.getValueType() == N->getValueType(0))
498 return BitConvertToInteger(Op);
499 }
500
501 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's
502 // entirely possible for both f16 and f32 to be legal, so use the fully
503 // hard-float FP_EXTEND rather than FP16_TO_FP.
504 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) {
505 if (IsStrict) {
506 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
507 { MVT::f32, MVT::Other }, { Chain, Op });
508 Chain = Op.getValue(1);
509 } else {
510 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
511 }
512 }
513
514 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
515 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
516 TargetLowering::MakeLibCallOptions CallOptions;
517 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
518 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
519 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
520 CallOptions, SDLoc(N),
521 Chain);
522 if (IsStrict)
523 ReplaceValueWith(SDValue(N, 1), Tmp.second);
524 return Tmp.first;
525 }
526
527 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
528 // nodes?
529 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
530 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
531 SDValue Op = N->getOperand(0);
532 TargetLowering::MakeLibCallOptions CallOptions;
533 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
534 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
535 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
536 CallOptions, SDLoc(N)).first;
537 if (N->getValueType(0) == MVT::f32)
538 return Res32;
539
540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
541 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
542 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
543 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
544 }
545
546 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
547 bool IsStrict = N->isStrictFPOpcode();
548 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
549 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
550 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
551 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
552 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
553 TargetLowering::MakeLibCallOptions CallOptions;
554 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
555 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
556 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
557 CallOptions, SDLoc(N),
558 Chain);
559 if (IsStrict)
560 ReplaceValueWith(SDValue(N, 1), Tmp.second);
561 return Tmp.first;
562 }
563
564 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
565 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
566 RTLIB::POW_F32,
567 RTLIB::POW_F64,
568 RTLIB::POW_F80,
569 RTLIB::POW_F128,
570 RTLIB::POW_PPCF128));
571 }
572
573 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
574 bool IsStrict = N->isStrictFPOpcode();
575 unsigned Offset = IsStrict ? 1 : 0;
576 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
577 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
578 "Unsupported power type!");
579 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
581 if (!TLI.getLibcallName(LC)) {
582 // Some targets don't have a powi libcall; use pow instead.
583 // FIXME: Implement this if some target needs it.
584 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
585 return DAG.getUNDEF(N->getValueType(0));
586 }
587
588 if (DAG.getLibInfo().getIntSize() !=
589 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
590 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
591 // would use the wrong type for the argument.
592 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
593 return DAG.getUNDEF(N->getValueType(0));
594 }
595
596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
597 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
598 N->getOperand(1 + Offset) };
599 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
600 TargetLowering::MakeLibCallOptions CallOptions;
601 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
602 N->getOperand(1 + Offset).getValueType() };
603 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
604 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
605 CallOptions, SDLoc(N),
606 Chain);
607 if (IsStrict)
608 ReplaceValueWith(SDValue(N, 1), Tmp.second);
609 return Tmp.first;
610 }
611
612 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
613 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
614 RTLIB::REM_F32,
615 RTLIB::REM_F64,
616 RTLIB::REM_F80,
617 RTLIB::REM_F128,
618 RTLIB::REM_PPCF128));
619 }
620
621 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
622 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
623 RTLIB::RINT_F32,
624 RTLIB::RINT_F64,
625 RTLIB::RINT_F80,
626 RTLIB::RINT_F128,
627 RTLIB::RINT_PPCF128));
628 }
629
630 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
631 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
632 RTLIB::ROUND_F32,
633 RTLIB::ROUND_F64,
634 RTLIB::ROUND_F80,
635 RTLIB::ROUND_F128,
636 RTLIB::ROUND_PPCF128));
637 }
638
639 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
640 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
641 RTLIB::ROUNDEVEN_F32,
642 RTLIB::ROUNDEVEN_F64,
643 RTLIB::ROUNDEVEN_F80,
644 RTLIB::ROUNDEVEN_F128,
645 RTLIB::ROUNDEVEN_PPCF128));
646 }
647
648 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
649 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
650 RTLIB::SIN_F32,
651 RTLIB::SIN_F64,
652 RTLIB::SIN_F80,
653 RTLIB::SIN_F128,
654 RTLIB::SIN_PPCF128));
655 }
656
657 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
658 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
659 RTLIB::SQRT_F32,
660 RTLIB::SQRT_F64,
661 RTLIB::SQRT_F80,
662 RTLIB::SQRT_F128,
663 RTLIB::SQRT_PPCF128));
664 }
665
666 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
667 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
668 RTLIB::SUB_F32,
669 RTLIB::SUB_F64,
670 RTLIB::SUB_F80,
671 RTLIB::SUB_F128,
672 RTLIB::SUB_PPCF128));
673 }
674
675 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
676 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
677 RTLIB::TRUNC_F32,
678 RTLIB::TRUNC_F64,
679 RTLIB::TRUNC_F80,
680 RTLIB::TRUNC_F128,
681 RTLIB::TRUNC_PPCF128));
682 }
683
684 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
685 LoadSDNode *L = cast<LoadSDNode>(N);
686 EVT VT = N->getValueType(0);
687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
688 SDLoc dl(N);
689
690 auto MMOFlags =
691 L->getMemOperand()->getFlags() &
692 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
693 SDValue NewL;
694 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
695 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
696 L->getChain(), L->getBasePtr(), L->getOffset(),
697 L->getPointerInfo(), NVT, L->getOriginalAlign(),
698 MMOFlags, L->getAAInfo());
699 // Legalized the chain result - switch anything that used the old chain to
700 // use the new one.
701 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
702 return NewL;
703 }
704
705 // Do a non-extending load followed by FP_EXTEND.
706 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
707 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
708 L->getPointerInfo(), L->getMemoryVT(),
709 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
710 // Legalized the chain result - switch anything that used the old chain to
711 // use the new one.
712 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
713 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
714 return BitConvertToInteger(ExtendNode);
715 }
716
717 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
718 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
719 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
720 return DAG.getSelect(SDLoc(N),
721 LHS.getValueType(), N->getOperand(0), LHS, RHS);
722 }
723
724 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
725 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
726 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
727 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
728 LHS.getValueType(), N->getOperand(0),
729 N->getOperand(1), LHS, RHS, N->getOperand(4));
730 }
731
732 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
733 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
734 N->getValueType(0)));
735 }
736
737 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
738 SDValue Chain = N->getOperand(0); // Get the chain.
739 SDValue Ptr = N->getOperand(1); // Get the pointer.
740 EVT VT = N->getValueType(0);
741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
742 SDLoc dl(N);
743
744 SDValue NewVAARG;
745 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
746 N->getConstantOperandVal(3));
747
748 // Legalized the chain result - switch anything that used the old chain to
749 // use the new one.
750 if (N != NewVAARG.getValue(1).getNode())
751 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
752 return NewVAARG;
753 }
754
755 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
756 bool IsStrict = N->isStrictFPOpcode();
757 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
758 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
759 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
760 EVT RVT = N->getValueType(0);
761 EVT NVT = EVT();
762 SDLoc dl(N);
763
764 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
765 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
766 // match. Look for an appropriate libcall.
767 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
768 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
769 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
770 NVT = (MVT::SimpleValueType)t;
771 // The source needs to big enough to hold the operand.
772 if (NVT.bitsGE(SVT))
773 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
774 }
775 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
776
777 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
778 // Sign/zero extend the argument if the libcall takes a larger type.
779 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
780 NVT, N->getOperand(IsStrict ? 1 : 0));
781 TargetLowering::MakeLibCallOptions CallOptions;
782 CallOptions.setSExt(Signed);
783 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
784 std::pair<SDValue, SDValue> Tmp =
785 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
786 Op, CallOptions, dl, Chain);
787
788 if (IsStrict)
789 ReplaceValueWith(SDValue(N, 1), Tmp.second);
790 return Tmp.first;
791 }
792
793 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
794 // Expand and soften recursively.
795 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
796 return SDValue();
797 }
798
799 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
800 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
801 return SDValue();
802 }
803
804 //===----------------------------------------------------------------------===//
805 // Convert Float Operand to Integer
806 //===----------------------------------------------------------------------===//
807
808 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
809 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
810 dbgs() << "\n");
811 SDValue Res = SDValue();
812
813 switch (N->getOpcode()) {
814 default:
815 #ifndef NDEBUG
816 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
817 N->dump(&DAG); dbgs() << "\n";
818 #endif
819 llvm_unreachable("Do not know how to soften this operator's operand!");
820
821 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
822 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
823 case ISD::STRICT_FP_TO_FP16:
824 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
825 case ISD::STRICT_FP_ROUND:
826 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
827 case ISD::STRICT_FP_TO_SINT:
828 case ISD::STRICT_FP_TO_UINT:
829 case ISD::FP_TO_SINT:
830 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
831 case ISD::FP_TO_SINT_SAT:
832 case ISD::FP_TO_UINT_SAT:
833 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
834 case ISD::STRICT_LROUND:
835 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
836 case ISD::STRICT_LLROUND:
837 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
838 case ISD::STRICT_LRINT:
839 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
840 case ISD::STRICT_LLRINT:
841 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
842 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
843 case ISD::STRICT_FSETCC:
844 case ISD::STRICT_FSETCCS:
845 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
846 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
847 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
848 }
849
850 // If the result is null, the sub-method took care of registering results etc.
851 if (!Res.getNode()) return false;
852
853 // If the result is N, the sub-method updated N in place. Tell the legalizer
854 // core about this to re-analyze.
855 if (Res.getNode() == N)
856 return true;
857
858 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
859 "Invalid operand softening");
860
861 ReplaceValueWith(SDValue(N, 0), Res);
862 return false;
863 }
864
865 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
866 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
867
868 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
869 }
870
871 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
872 // We actually deal with the partially-softened FP_TO_FP16 node too, which
873 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
874 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
875 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
876 N->getOpcode() == ISD::STRICT_FP_ROUND);
877
878 bool IsStrict = N->isStrictFPOpcode();
879 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
880 EVT SVT = Op.getValueType();
881 EVT RVT = N->getValueType(0);
882 EVT FloatRVT = (N->getOpcode() == ISD::FP_TO_FP16 ||
883 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
884 ? MVT::f16
885 : RVT;
886
887 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
888 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
889
890 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
891 Op = GetSoftenedFloat(Op);
892 TargetLowering::MakeLibCallOptions CallOptions;
893 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
894 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
895 CallOptions, SDLoc(N),
896 Chain);
897 if (IsStrict) {
898 ReplaceValueWith(SDValue(N, 1), Tmp.second);
899 ReplaceValueWith(SDValue(N, 0), Tmp.first);
900 return SDValue();
901 }
902 return Tmp.first;
903 }
904
905 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
906 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
907 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
908
909 EVT VT = NewLHS.getValueType();
910 NewLHS = GetSoftenedFloat(NewLHS);
911 NewRHS = GetSoftenedFloat(NewRHS);
912 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
913 N->getOperand(2), N->getOperand(3));
914
915 // If softenSetCCOperands returned a scalar, we need to compare the result
916 // against zero to select between true and false values.
917 if (!NewRHS.getNode()) {
918 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
919 CCCode = ISD::SETNE;
920 }
921
922 // Update N to have the operands specified.
923 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
924 DAG.getCondCode(CCCode), NewLHS, NewRHS,
925 N->getOperand(4)),
926 0);
927 }
928
929 // Even if the result type is legal, no libcall may exactly match. (e.g. We
930 // don't have FP-i8 conversions) This helper method looks for an appropriate
931 // promoted libcall.
932 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
933 bool Signed) {
934 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
935 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
936 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
937 ++IntVT) {
938 Promoted = (MVT::SimpleValueType)IntVT;
939 // The type needs to big enough to hold the result.
940 if (Promoted.bitsGE(RetVT))
941 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
942 : RTLIB::getFPTOUINT(SrcVT, Promoted);
943 }
944 return LC;
945 }
946
947 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
948 bool IsStrict = N->isStrictFPOpcode();
949 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
950 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
951
952 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
953 EVT SVT = Op.getValueType();
954 EVT RVT = N->getValueType(0);
955 EVT NVT = EVT();
956 SDLoc dl(N);
957
958 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
959 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
960 // match, eg. we don't have fp -> i8 conversions.
961 // Look for an appropriate libcall.
962 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
963 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
964 "Unsupported FP_TO_XINT!");
965
966 Op = GetSoftenedFloat(Op);
967 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
968 TargetLowering::MakeLibCallOptions CallOptions;
969 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
970 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
971 CallOptions, dl, Chain);
972
973 // Truncate the result if the libcall returns a larger type.
974 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
975
976 if (!IsStrict)
977 return Res;
978
979 ReplaceValueWith(SDValue(N, 1), Tmp.second);
980 ReplaceValueWith(SDValue(N, 0), Res);
981 return SDValue();
982 }
983
984 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
985 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
986 return Res;
987 }
988
989 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
990 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
991 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
992
993 EVT VT = NewLHS.getValueType();
994 NewLHS = GetSoftenedFloat(NewLHS);
995 NewRHS = GetSoftenedFloat(NewRHS);
996 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
997 N->getOperand(0), N->getOperand(1));
998
999 // If softenSetCCOperands returned a scalar, we need to compare the result
1000 // against zero to select between true and false values.
1001 if (!NewRHS.getNode()) {
1002 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1003 CCCode = ISD::SETNE;
1004 }
1005
1006 // Update N to have the operands specified.
1007 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1008 N->getOperand(2), N->getOperand(3),
1009 DAG.getCondCode(CCCode)),
1010 0);
1011 }
1012
1013 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1014 bool IsStrict = N->isStrictFPOpcode();
1015 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1016 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1017 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1018 ISD::CondCode CCCode =
1019 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1020
1021 EVT VT = Op0.getValueType();
1022 SDValue NewLHS = GetSoftenedFloat(Op0);
1023 SDValue NewRHS = GetSoftenedFloat(Op1);
1024 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1025 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1026
1027 // Update N to have the operands specified.
1028 if (NewRHS.getNode()) {
1029 if (IsStrict)
1030 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1031 NewRHS, DAG.getCondCode(CCCode));
1032 else
1033 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1034 DAG.getCondCode(CCCode)), 0);
1035 }
1036
1037 // Otherwise, softenSetCCOperands returned a scalar, use it.
1038 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1039 "Unexpected setcc expansion!");
1040
1041 if (IsStrict) {
1042 ReplaceValueWith(SDValue(N, 0), NewLHS);
1043 ReplaceValueWith(SDValue(N, 1), Chain);
1044 return SDValue();
1045 }
1046 return NewLHS;
1047 }
1048
1049 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1050 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1051 assert(OpNo == 1 && "Can only soften the stored value!");
1052 StoreSDNode *ST = cast<StoreSDNode>(N);
1053 SDValue Val = ST->getValue();
1054 SDLoc dl(N);
1055
1056 if (ST->isTruncatingStore())
1057 // Do an FP_ROUND followed by a non-truncating store.
1058 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
1059 Val, DAG.getIntPtrConstant(0, dl)));
1060 else
1061 Val = GetSoftenedFloat(Val);
1062
1063 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1064 ST->getMemOperand());
1065 }
1066
1067 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1068 SDValue LHS = N->getOperand(0);
1069 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1070 SDLoc dl(N);
1071
1072 EVT LVT = LHS.getValueType();
1073 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1074 EVT RVT = RHS.getValueType();
1075
1076 unsigned LSize = LVT.getSizeInBits();
1077 unsigned RSize = RVT.getSizeInBits();
1078
1079 // Shift right or sign-extend it if the two operands have different types.
1080 int SizeDiff = RSize - LSize;
1081 if (SizeDiff > 0) {
1082 RHS =
1083 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1084 DAG.getConstant(SizeDiff, dl,
1085 TLI.getShiftAmountTy(RHS.getValueType(),
1086 DAG.getDataLayout())));
1087 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1088 } else if (SizeDiff < 0) {
1089 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1090 RHS =
1091 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1092 DAG.getConstant(-SizeDiff, dl,
1093 TLI.getShiftAmountTy(RHS.getValueType(),
1094 DAG.getDataLayout())));
1095 }
1096
1097 RHS = DAG.getBitcast(LVT, RHS);
1098 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1099 }
1100
1101 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1102 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1103 bool IsStrict = N->isStrictFPOpcode();
1104 unsigned Offset = IsStrict ? 1 : 0;
1105 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1106 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1107 TargetLowering::MakeLibCallOptions CallOptions;
1108 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1109 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1110 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1111 CallOptions, SDLoc(N),
1112 Chain);
1113 if (IsStrict) {
1114 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1115 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1116 return SDValue();
1117 }
1118
1119 return Tmp.first;
1120 }
1121
1122 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1123 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1124 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1125 RTLIB::LROUND_F32,
1126 RTLIB::LROUND_F64,
1127 RTLIB::LROUND_F80,
1128 RTLIB::LROUND_F128,
1129 RTLIB::LROUND_PPCF128));
1130 }
1131
1132 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1133 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1134 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1135 RTLIB::LLROUND_F32,
1136 RTLIB::LLROUND_F64,
1137 RTLIB::LLROUND_F80,
1138 RTLIB::LLROUND_F128,
1139 RTLIB::LLROUND_PPCF128));
1140 }
1141
1142 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1143 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1144 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1145 RTLIB::LRINT_F32,
1146 RTLIB::LRINT_F64,
1147 RTLIB::LRINT_F80,
1148 RTLIB::LRINT_F128,
1149 RTLIB::LRINT_PPCF128));
1150 }
1151
1152 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1153 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1154 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1155 RTLIB::LLRINT_F32,
1156 RTLIB::LLRINT_F64,
1157 RTLIB::LLRINT_F80,
1158 RTLIB::LLRINT_F128,
1159 RTLIB::LLRINT_PPCF128));
1160 }
1161
1162 //===----------------------------------------------------------------------===//
1163 // Float Result Expansion
1164 //===----------------------------------------------------------------------===//
1165
1166 /// ExpandFloatResult - This method is called when the specified result of the
1167 /// specified node is found to need expansion. At this point, the node may also
1168 /// have invalid operands or may have other results that need promotion, we just
1169 /// know that (at least) one result needs expansion.
1170 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1171 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1172 SDValue Lo, Hi;
1173 Lo = Hi = SDValue();
1174
1175 // See if the target wants to custom expand this node.
1176 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1177 return;
1178
1179 switch (N->getOpcode()) {
1180 default:
1181 #ifndef NDEBUG
1182 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1183 N->dump(&DAG); dbgs() << "\n";
1184 #endif
1185 llvm_unreachable("Do not know how to expand the result of this operator!");
1186
1187 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1188 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1189 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1190
1191 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1192 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1193 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1194 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1195 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1196 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1197
1198 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1199 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1200 case ISD::STRICT_FMINNUM:
1201 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1202 case ISD::STRICT_FMAXNUM:
1203 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1204 case ISD::STRICT_FADD:
1205 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1206 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1207 case ISD::STRICT_FCEIL:
1208 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1209 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1210 case ISD::STRICT_FCOS:
1211 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1212 case ISD::STRICT_FDIV:
1213 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1214 case ISD::STRICT_FEXP:
1215 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1216 case ISD::STRICT_FEXP2:
1217 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1218 case ISD::STRICT_FFLOOR:
1219 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1220 case ISD::STRICT_FLOG:
1221 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1222 case ISD::STRICT_FLOG2:
1223 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1224 case ISD::STRICT_FLOG10:
1225 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1226 case ISD::STRICT_FMA:
1227 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1228 case ISD::STRICT_FMUL:
1229 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1230 case ISD::STRICT_FNEARBYINT:
1231 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1232 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1233 case ISD::STRICT_FP_EXTEND:
1234 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1235 case ISD::STRICT_FPOW:
1236 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1237 case ISD::STRICT_FPOWI:
1238 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1239 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1240 case ISD::STRICT_FRINT:
1241 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1242 case ISD::STRICT_FROUND:
1243 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1244 case ISD::STRICT_FROUNDEVEN:
1245 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1246 case ISD::STRICT_FSIN:
1247 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1248 case ISD::STRICT_FSQRT:
1249 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1250 case ISD::STRICT_FSUB:
1251 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1252 case ISD::STRICT_FTRUNC:
1253 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1254 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1255 case ISD::STRICT_SINT_TO_FP:
1256 case ISD::STRICT_UINT_TO_FP:
1257 case ISD::SINT_TO_FP:
1258 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1259 case ISD::STRICT_FREM:
1260 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1261 }
1262
1263 // If Lo/Hi is null, the sub-method took care of registering results etc.
1264 if (Lo.getNode())
1265 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1266 }
1267
1268 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1269 SDValue &Hi) {
1270 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1271 assert(NVT.getSizeInBits() == 64 &&
1272 "Do not know how to expand this float constant!");
1273 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1274 SDLoc dl(N);
1275 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1276 APInt(64, C.getRawData()[1])),
1277 dl, NVT);
1278 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1279 APInt(64, C.getRawData()[0])),
1280 dl, NVT);
1281 }
1282
1283 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1284 SDValue &Lo, SDValue &Hi) {
1285 bool IsStrict = N->isStrictFPOpcode();
1286 unsigned Offset = IsStrict ? 1 : 0;
1287 SDValue Op = N->getOperand(0 + Offset);
1288 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1289 TargetLowering::MakeLibCallOptions CallOptions;
1290 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1291 Op, CallOptions, SDLoc(N),
1292 Chain);
1293 if (IsStrict)
1294 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1295 GetPairElements(Tmp.first, Lo, Hi);
1296 }
1297
1298 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1299 SDValue &Lo, SDValue &Hi) {
1300 bool IsStrict = N->isStrictFPOpcode();
1301 unsigned Offset = IsStrict ? 1 : 0;
1302 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1303 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1304 TargetLowering::MakeLibCallOptions CallOptions;
1305 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1306 Ops, CallOptions, SDLoc(N),
1307 Chain);
1308 if (IsStrict)
1309 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1310 GetPairElements(Tmp.first, Lo, Hi);
1311 }
1312
1313 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1314 SDValue &Hi) {
1315 assert(N->getValueType(0) == MVT::ppcf128 &&
1316 "Logic only correct for ppcf128!");
1317 SDLoc dl(N);
1318 SDValue Tmp;
1319 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1320 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1321 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1322 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1323 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1324 ISD::SETEQ);
1325 }
1326
1327 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1328 SDValue &Hi) {
1329 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1330 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1331 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1332 RTLIB::FMIN_PPCF128), Lo, Hi);
1333 }
1334
1335 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1336 SDValue &Hi) {
1337 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1338 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1339 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1340 RTLIB::FMAX_PPCF128), Lo, Hi);
1341 }
1342
1343 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1344 SDValue &Hi) {
1345 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1346 RTLIB::ADD_F32, RTLIB::ADD_F64,
1347 RTLIB::ADD_F80, RTLIB::ADD_F128,
1348 RTLIB::ADD_PPCF128), Lo, Hi);
1349 }
1350
1351 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1352 SDValue &Hi) {
1353 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1354 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1355 RTLIB::CBRT_F128,
1356 RTLIB::CBRT_PPCF128), Lo, Hi);
1357 }
1358
1359 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1360 SDValue &Lo, SDValue &Hi) {
1361 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1362 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1363 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1364 RTLIB::CEIL_PPCF128), Lo, Hi);
1365 }
1366
1367 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1368 SDValue &Lo, SDValue &Hi) {
1369 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1370 RTLIB::COPYSIGN_F32,
1371 RTLIB::COPYSIGN_F64,
1372 RTLIB::COPYSIGN_F80,
1373 RTLIB::COPYSIGN_F128,
1374 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1375 }
1376
1377 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1378 SDValue &Lo, SDValue &Hi) {
1379 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1380 RTLIB::COS_F32, RTLIB::COS_F64,
1381 RTLIB::COS_F80, RTLIB::COS_F128,
1382 RTLIB::COS_PPCF128), Lo, Hi);
1383 }
1384
1385 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1386 SDValue &Hi) {
1387 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1388 RTLIB::DIV_F32,
1389 RTLIB::DIV_F64,
1390 RTLIB::DIV_F80,
1391 RTLIB::DIV_F128,
1392 RTLIB::DIV_PPCF128), Lo, Hi);
1393 }
1394
1395 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1396 SDValue &Lo, SDValue &Hi) {
1397 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1398 RTLIB::EXP_F32, RTLIB::EXP_F64,
1399 RTLIB::EXP_F80, RTLIB::EXP_F128,
1400 RTLIB::EXP_PPCF128), Lo, Hi);
1401 }
1402
1403 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1404 SDValue &Lo, SDValue &Hi) {
1405 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1406 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1407 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1408 RTLIB::EXP2_PPCF128), Lo, Hi);
1409 }
1410
1411 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1412 SDValue &Lo, SDValue &Hi) {
1413 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1414 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1415 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1416 RTLIB::FLOOR_PPCF128), Lo, Hi);
1417 }
1418
1419 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1420 SDValue &Lo, SDValue &Hi) {
1421 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1422 RTLIB::LOG_F32, RTLIB::LOG_F64,
1423 RTLIB::LOG_F80, RTLIB::LOG_F128,
1424 RTLIB::LOG_PPCF128), Lo, Hi);
1425 }
1426
1427 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1428 SDValue &Lo, SDValue &Hi) {
1429 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1430 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1431 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1432 RTLIB::LOG2_PPCF128), Lo, Hi);
1433 }
1434
1435 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1436 SDValue &Lo, SDValue &Hi) {
1437 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1438 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1439 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1440 RTLIB::LOG10_PPCF128), Lo, Hi);
1441 }
1442
1443 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1444 SDValue &Hi) {
1445 bool IsStrict = N->isStrictFPOpcode();
1446 unsigned Offset = IsStrict ? 1 : 0;
1447 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1448 N->getOperand(2 + Offset) };
1449 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1450 TargetLowering::MakeLibCallOptions CallOptions;
1451 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1452 RTLIB::FMA_F32,
1453 RTLIB::FMA_F64,
1454 RTLIB::FMA_F80,
1455 RTLIB::FMA_F128,
1456 RTLIB::FMA_PPCF128),
1457 N->getValueType(0), Ops, CallOptions,
1458 SDLoc(N), Chain);
1459 if (IsStrict)
1460 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1461 GetPairElements(Tmp.first, Lo, Hi);
1462 }
1463
1464 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1465 SDValue &Hi) {
1466 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1467 RTLIB::MUL_F32,
1468 RTLIB::MUL_F64,
1469 RTLIB::MUL_F80,
1470 RTLIB::MUL_F128,
1471 RTLIB::MUL_PPCF128), Lo, Hi);
1472 }
1473
1474 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1475 SDValue &Lo, SDValue &Hi) {
1476 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1477 RTLIB::NEARBYINT_F32,
1478 RTLIB::NEARBYINT_F64,
1479 RTLIB::NEARBYINT_F80,
1480 RTLIB::NEARBYINT_F128,
1481 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1482 }
1483
1484 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1485 SDValue &Hi) {
1486 SDLoc dl(N);
1487 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1488 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1489 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1490 }
1491
1492 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1493 SDValue &Hi) {
1494 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1495 SDLoc dl(N);
1496 bool IsStrict = N->isStrictFPOpcode();
1497
1498 SDValue Chain;
1499 if (IsStrict) {
1500 // If the expanded type is the same as the input type, just bypass the node.
1501 if (NVT == N->getOperand(1).getValueType()) {
1502 Hi = N->getOperand(1);
1503 Chain = N->getOperand(0);
1504 } else {
1505 // Other we need to extend.
1506 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1507 { N->getOperand(0), N->getOperand(1) });
1508 Chain = Hi.getValue(1);
1509 }
1510 } else {
1511 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1512 }
1513
1514 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1515 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1516
1517 if (IsStrict)
1518 ReplaceValueWith(SDValue(N, 1), Chain);
1519 }
1520
1521 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1522 SDValue &Lo, SDValue &Hi) {
1523 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1524 RTLIB::POW_F32, RTLIB::POW_F64,
1525 RTLIB::POW_F80, RTLIB::POW_F128,
1526 RTLIB::POW_PPCF128), Lo, Hi);
1527 }
1528
1529 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1530 SDValue &Lo, SDValue &Hi) {
1531 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1532 }
1533
1534 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1535 SDValue &Lo, SDValue &Hi) {
1536 assert(N->getValueType(0) == MVT::ppcf128 &&
1537 "Logic only correct for ppcf128!");
1538
1539 SDLoc dl(N);
1540 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1541 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1542 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1543 }
1544
1545 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1546 SDValue &Lo, SDValue &Hi) {
1547 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1548 RTLIB::REM_F32, RTLIB::REM_F64,
1549 RTLIB::REM_F80, RTLIB::REM_F128,
1550 RTLIB::REM_PPCF128), Lo, Hi);
1551 }
1552
1553 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1554 SDValue &Lo, SDValue &Hi) {
1555 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1556 RTLIB::RINT_F32, RTLIB::RINT_F64,
1557 RTLIB::RINT_F80, RTLIB::RINT_F128,
1558 RTLIB::RINT_PPCF128), Lo, Hi);
1559 }
1560
1561 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1562 SDValue &Lo, SDValue &Hi) {
1563 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1564 RTLIB::ROUND_F32,
1565 RTLIB::ROUND_F64,
1566 RTLIB::ROUND_F80,
1567 RTLIB::ROUND_F128,
1568 RTLIB::ROUND_PPCF128), Lo, Hi);
1569 }
1570
1571 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1572 SDValue &Lo, SDValue &Hi) {
1573 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1574 RTLIB::ROUNDEVEN_F32,
1575 RTLIB::ROUNDEVEN_F64,
1576 RTLIB::ROUNDEVEN_F80,
1577 RTLIB::ROUNDEVEN_F128,
1578 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1579 }
1580
1581 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1582 SDValue &Lo, SDValue &Hi) {
1583 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1584 RTLIB::SIN_F32, RTLIB::SIN_F64,
1585 RTLIB::SIN_F80, RTLIB::SIN_F128,
1586 RTLIB::SIN_PPCF128), Lo, Hi);
1587 }
1588
1589 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1590 SDValue &Lo, SDValue &Hi) {
1591 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1592 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1593 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1594 RTLIB::SQRT_PPCF128), Lo, Hi);
1595 }
1596
1597 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1598 SDValue &Hi) {
1599 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1600 RTLIB::SUB_F32,
1601 RTLIB::SUB_F64,
1602 RTLIB::SUB_F80,
1603 RTLIB::SUB_F128,
1604 RTLIB::SUB_PPCF128), Lo, Hi);
1605 }
1606
1607 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1608 SDValue &Lo, SDValue &Hi) {
1609 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1610 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1611 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1612 RTLIB::TRUNC_PPCF128), Lo, Hi);
1613 }
1614
1615 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1616 SDValue &Hi) {
1617 if (ISD::isNormalLoad(N)) {
1618 ExpandRes_NormalLoad(N, Lo, Hi);
1619 return;
1620 }
1621
1622 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1623 LoadSDNode *LD = cast<LoadSDNode>(N);
1624 SDValue Chain = LD->getChain();
1625 SDValue Ptr = LD->getBasePtr();
1626 SDLoc dl(N);
1627
1628 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1629 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1630 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1631
1632 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1633 LD->getMemoryVT(), LD->getMemOperand());
1634
1635 // Remember the chain.
1636 Chain = Hi.getValue(1);
1637
1638 // The low part is zero.
1639 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1640 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1641
1642 // Modified the chain - switch anything that used the old chain to use the
1643 // new one.
1644 ReplaceValueWith(SDValue(LD, 1), Chain);
1645 }
1646
1647 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1648 SDValue &Hi) {
1649 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1650 EVT VT = N->getValueType(0);
1651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1652 bool Strict = N->isStrictFPOpcode();
1653 SDValue Src = N->getOperand(Strict ? 1 : 0);
1654 EVT SrcVT = Src.getValueType();
1655 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1656 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1657 SDLoc dl(N);
1658 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1659
1660 // TODO: Any other flags to propagate?
1661 SDNodeFlags Flags;
1662 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1663
1664 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1665 // When promoting partial word types to i32 we must honor the signedness,
1666 // though.
1667 if (SrcVT.bitsLE(MVT::i32)) {
1668 // The integer can be represented exactly in an f64.
1669 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1670 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1671 if (Strict) {
1672 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1673 {Chain, Src}, Flags);
1674 Chain = Hi.getValue(1);
1675 } else
1676 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1677 } else {
1678 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1679 if (SrcVT.bitsLE(MVT::i64)) {
1680 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1681 MVT::i64, Src);
1682 LC = RTLIB::SINTTOFP_I64_PPCF128;
1683 } else if (SrcVT.bitsLE(MVT::i128)) {
1684 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1685 LC = RTLIB::SINTTOFP_I128_PPCF128;
1686 }
1687 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1688
1689 TargetLowering::MakeLibCallOptions CallOptions;
1690 CallOptions.setSExt(true);
1691 std::pair<SDValue, SDValue> Tmp =
1692 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1693 if (Strict)
1694 Chain = Tmp.second;
1695 GetPairElements(Tmp.first, Lo, Hi);
1696 }
1697
1698 // No need to complement for unsigned 32-bit integers
1699 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1700 if (Strict)
1701 ReplaceValueWith(SDValue(N, 1), Chain);
1702
1703 return;
1704 }
1705
1706 // Unsigned - fix up the SINT_TO_FP value just calculated.
1707 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1708 // keep semantics correctness if the integer is not exactly representable
1709 // here. See ExpandLegalINT_TO_FP.
1710 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1711 SrcVT = Src.getValueType();
1712
1713 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1714 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1715 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1716 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1717 ArrayRef<uint64_t> Parts;
1718
1719 switch (SrcVT.getSimpleVT().SimpleTy) {
1720 default:
1721 llvm_unreachable("Unsupported UINT_TO_FP!");
1722 case MVT::i32:
1723 Parts = TwoE32;
1724 break;
1725 case MVT::i64:
1726 Parts = TwoE64;
1727 break;
1728 case MVT::i128:
1729 Parts = TwoE128;
1730 break;
1731 }
1732
1733 // TODO: Are there other fast-math-flags to propagate to this FADD?
1734 SDValue NewLo = DAG.getConstantFP(
1735 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1736 if (Strict) {
1737 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1738 {Chain, Hi, NewLo}, Flags);
1739 Chain = Lo.getValue(1);
1740 ReplaceValueWith(SDValue(N, 1), Chain);
1741 } else
1742 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1743 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1744 Lo, Hi, ISD::SETLT);
1745 GetPairElements(Lo, Lo, Hi);
1746 }
1747
1748
1749 //===----------------------------------------------------------------------===//
1750 // Float Operand Expansion
1751 //===----------------------------------------------------------------------===//
1752
1753 /// ExpandFloatOperand - This method is called when the specified operand of the
1754 /// specified node is found to need expansion. At this point, all of the result
1755 /// types of the node are known to be legal, but other operands of the node may
1756 /// need promotion or expansion as well as the specified one.
1757 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1758 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1759 SDValue Res = SDValue();
1760
1761 // See if the target wants to custom expand this node.
1762 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1763 return false;
1764
1765 switch (N->getOpcode()) {
1766 default:
1767 #ifndef NDEBUG
1768 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1769 N->dump(&DAG); dbgs() << "\n";
1770 #endif
1771 llvm_unreachable("Do not know how to expand this operator's operand!");
1772
1773 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1774 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1775 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1776
1777 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1778 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1779 case ISD::STRICT_FP_ROUND:
1780 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1781 case ISD::STRICT_FP_TO_SINT:
1782 case ISD::STRICT_FP_TO_UINT:
1783 case ISD::FP_TO_SINT:
1784 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1785 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1786 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1787 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1788 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1789 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1790 case ISD::STRICT_FSETCC:
1791 case ISD::STRICT_FSETCCS:
1792 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1793 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1794 OpNo); break;
1795 }
1796
1797 // If the result is null, the sub-method took care of registering results etc.
1798 if (!Res.getNode()) return false;
1799
1800 // If the result is N, the sub-method updated N in place. Tell the legalizer
1801 // core about this.
1802 if (Res.getNode() == N)
1803 return true;
1804
1805 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1806 "Invalid operand expansion");
1807
1808 ReplaceValueWith(SDValue(N, 0), Res);
1809 return false;
1810 }
1811
1812 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1813 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1814 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1815 SDValue &NewRHS,
1816 ISD::CondCode &CCCode,
1817 const SDLoc &dl, SDValue &Chain,
1818 bool IsSignaling) {
1819 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1820 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1821 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1822
1823 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1824
1825 // FIXME: This generated code sucks. We want to generate
1826 // FCMPU crN, hi1, hi2
1827 // BNE crN, L:
1828 // FCMPU crN, lo1, lo2
1829 // The following can be improved, but not that much.
1830 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1831 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1832 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1833 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1834 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1835 RHSLo, CCCode, OutputChain, IsSignaling);
1836 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1837 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1838 Tmp1 =
1839 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1840 ISD::SETUNE, OutputChain, IsSignaling);
1841 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1842 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1843 RHSHi, CCCode, OutputChain, IsSignaling);
1844 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1845 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1846 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1847 NewRHS = SDValue(); // LHS is the result, not a compare.
1848 Chain = OutputChain;
1849 }
1850
1851 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1852 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1853 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1854 SDValue Chain;
1855 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1856
1857 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1858 // against zero to select between true and false values.
1859 if (!NewRHS.getNode()) {
1860 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1861 CCCode = ISD::SETNE;
1862 }
1863
1864 // Update N to have the operands specified.
1865 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1866 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1867 N->getOperand(4)), 0);
1868 }
1869
1870 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1871 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1872 "Logic only correct for ppcf128!");
1873 SDValue Lo, Hi;
1874 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1875 // The ppcf128 value is providing only the sign; take it from the
1876 // higher-order double (which must have the larger magnitude).
1877 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1878 N->getValueType(0), N->getOperand(0), Hi);
1879 }
1880
1881 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1882 bool IsStrict = N->isStrictFPOpcode();
1883 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1884 "Logic only correct for ppcf128!");
1885 SDValue Lo, Hi;
1886 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1887
1888 if (!IsStrict)
1889 // Round it the rest of the way (e.g. to f32) if needed.
1890 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1891 N->getValueType(0), Hi, N->getOperand(1));
1892
1893 // Eliminate the node if the input float type is the same as the output float
1894 // type.
1895 if (Hi.getValueType() == N->getValueType(0)) {
1896 // Connect the output chain to the input chain, unlinking the node.
1897 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1898 ReplaceValueWith(SDValue(N, 0), Hi);
1899 return SDValue();
1900 }
1901
1902 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1903 {N->getValueType(0), MVT::Other},
1904 {N->getOperand(0), Hi, N->getOperand(2)});
1905 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1906 ReplaceValueWith(SDValue(N, 0), Expansion);
1907 return SDValue();
1908 }
1909
1910 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1911 EVT RVT = N->getValueType(0);
1912 SDLoc dl(N);
1913
1914 bool IsStrict = N->isStrictFPOpcode();
1915 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1916 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1917 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1918 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1919
1920 EVT NVT;
1921 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1922 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1923 "Unsupported FP_TO_XINT!");
1924 TargetLowering::MakeLibCallOptions CallOptions;
1925 std::pair<SDValue, SDValue> Tmp =
1926 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1927 if (!IsStrict)
1928 return Tmp.first;
1929
1930 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1931 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1932 return SDValue();
1933 }
1934
1935 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1936 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1937 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1938 SDValue Chain;
1939 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1940
1941 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1942 // against zero to select between true and false values.
1943 if (!NewRHS.getNode()) {
1944 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1945 CCCode = ISD::SETNE;
1946 }
1947
1948 // Update N to have the operands specified.
1949 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1950 N->getOperand(2), N->getOperand(3),
1951 DAG.getCondCode(CCCode)), 0);
1952 }
1953
1954 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1955 bool IsStrict = N->isStrictFPOpcode();
1956 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1957 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1958 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1959 ISD::CondCode CCCode =
1960 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1961 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
1962 N->getOpcode() == ISD::STRICT_FSETCCS);
1963
1964 // FloatExpandSetCCOperands always returned a scalar.
1965 assert(!NewRHS.getNode() && "Expect to return scalar");
1966 assert(NewLHS.getValueType() == N->getValueType(0) &&
1967 "Unexpected setcc expansion!");
1968 if (Chain) {
1969 ReplaceValueWith(SDValue(N, 0), NewLHS);
1970 ReplaceValueWith(SDValue(N, 1), Chain);
1971 return SDValue();
1972 }
1973 return NewLHS;
1974 }
1975
1976 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1977 if (ISD::isNormalStore(N))
1978 return ExpandOp_NormalStore(N, OpNo);
1979
1980 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1981 assert(OpNo == 1 && "Can only expand the stored value so far");
1982 StoreSDNode *ST = cast<StoreSDNode>(N);
1983
1984 SDValue Chain = ST->getChain();
1985 SDValue Ptr = ST->getBasePtr();
1986
1987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
1988 ST->getValue().getValueType());
1989 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1990 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1991 (void)NVT;
1992
1993 SDValue Lo, Hi;
1994 GetExpandedOp(ST->getValue(), Lo, Hi);
1995
1996 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
1997 ST->getMemoryVT(), ST->getMemOperand());
1998 }
1999
2000 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2001 EVT RVT = N->getValueType(0);
2002 EVT RetVT = N->getOperand(0).getValueType();
2003 TargetLowering::MakeLibCallOptions CallOptions;
2004 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2005 RTLIB::LROUND_F32,
2006 RTLIB::LROUND_F64,
2007 RTLIB::LROUND_F80,
2008 RTLIB::LROUND_F128,
2009 RTLIB::LROUND_PPCF128),
2010 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2011 }
2012
2013 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2014 EVT RVT = N->getValueType(0);
2015 EVT RetVT = N->getOperand(0).getValueType();
2016 TargetLowering::MakeLibCallOptions CallOptions;
2017 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2018 RTLIB::LLROUND_F32,
2019 RTLIB::LLROUND_F64,
2020 RTLIB::LLROUND_F80,
2021 RTLIB::LLROUND_F128,
2022 RTLIB::LLROUND_PPCF128),
2023 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2024 }
2025
2026 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2027 EVT RVT = N->getValueType(0);
2028 EVT RetVT = N->getOperand(0).getValueType();
2029 TargetLowering::MakeLibCallOptions CallOptions;
2030 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2031 RTLIB::LRINT_F32,
2032 RTLIB::LRINT_F64,
2033 RTLIB::LRINT_F80,
2034 RTLIB::LRINT_F128,
2035 RTLIB::LRINT_PPCF128),
2036 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2037 }
2038
2039 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2040 EVT RVT = N->getValueType(0);
2041 EVT RetVT = N->getOperand(0).getValueType();
2042 TargetLowering::MakeLibCallOptions CallOptions;
2043 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2044 RTLIB::LLRINT_F32,
2045 RTLIB::LLRINT_F64,
2046 RTLIB::LLRINT_F80,
2047 RTLIB::LLRINT_F128,
2048 RTLIB::LLRINT_PPCF128),
2049 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2050 }
2051
2052 //===----------------------------------------------------------------------===//
2053 // Float Operand Promotion
2054 //===----------------------------------------------------------------------===//
2055 //
2056
2057 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2058 if (OpVT == MVT::f16) {
2059 return ISD::FP16_TO_FP;
2060 } else if (RetVT == MVT::f16) {
2061 return ISD::FP_TO_FP16;
2062 }
2063
2064 report_fatal_error("Attempt at an invalid promotion-related conversion");
2065 }
2066
2067 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2068 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2069 dbgs() << "\n");
2070 SDValue R = SDValue();
2071
2072 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2073 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2074 return false;
2075 }
2076
2077 // Nodes that use a promotion-requiring floating point operand, but doesn't
2078 // produce a promotion-requiring floating point result, need to be legalized
2079 // to use the promoted float operand. Nodes that produce at least one
2080 // promotion-requiring floating point result have their operands legalized as
2081 // a part of PromoteFloatResult.
2082 switch (N->getOpcode()) {
2083 default:
2084 #ifndef NDEBUG
2085 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2086 N->dump(&DAG); dbgs() << "\n";
2087 #endif
2088 llvm_unreachable("Do not know how to promote this operator's operand!");
2089
2090 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2091 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2092 case ISD::FP_TO_SINT:
2093 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2094 case ISD::FP_TO_SINT_SAT:
2095 case ISD::FP_TO_UINT_SAT:
2096 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2097 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2098 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2099 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2100 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2101 }
2102
2103 if (R.getNode())
2104 ReplaceValueWith(SDValue(N, 0), R);
2105 return false;
2106 }
2107
2108 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2109 SDValue Op = N->getOperand(0);
2110 EVT OpVT = Op->getValueType(0);
2111
2112 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2113 EVT PromotedVT = Promoted->getValueType(0);
2114
2115 // Convert the promoted float value to the desired IVT.
2116 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2117 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2118 IVT, Promoted);
2119 // The final result type might not be an scalar so we need a bitcast. The
2120 // bitcast will be further legalized if needed.
2121 return DAG.getBitcast(N->getValueType(0), Convert);
2122 }
2123
2124 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2125 // PromoteFloatRes_FCOPYSIGN.
2126 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2127 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2128 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2129
2130 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2131 N->getOperand(0), Op1);
2132 }
2133
2134 // Convert the promoted float value to the desired integer type
2135 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2136 SDValue Op = GetPromotedFloat(N->getOperand(0));
2137 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2138 }
2139
2140 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2141 unsigned OpNo) {
2142 SDValue Op = GetPromotedFloat(N->getOperand(0));
2143 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2144 N->getOperand(1));
2145 }
2146
2147 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2148 SDValue Op = GetPromotedFloat(N->getOperand(0));
2149 EVT VT = N->getValueType(0);
2150
2151 // Desired VT is same as promoted type. Use promoted float directly.
2152 if (VT == Op->getValueType(0))
2153 return Op;
2154
2155 // Else, extend the promoted float value to the desired VT.
2156 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2157 }
2158
2159 // Promote the float operands used for comparison. The true- and false-
2160 // operands have the same type as the result and are promoted, if needed, by
2161 // PromoteFloatRes_SELECT_CC
2162 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2163 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2164 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2165
2166 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2167 LHS, RHS, N->getOperand(2), N->getOperand(3),
2168 N->getOperand(4));
2169 }
2170
2171 // Construct a SETCC that compares the promoted values and sets the conditional
2172 // code.
2173 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2174 EVT VT = N->getValueType(0);
2175 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2176 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2177 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2178
2179 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2180
2181 }
2182
2183 // Lower the promoted Float down to the integer value of same size and construct
2184 // a STORE of the integer value.
2185 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2186 StoreSDNode *ST = cast<StoreSDNode>(N);
2187 SDValue Val = ST->getValue();
2188 SDLoc DL(N);
2189
2190 SDValue Promoted = GetPromotedFloat(Val);
2191 EVT VT = ST->getOperand(1).getValueType();
2192 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2193
2194 SDValue NewVal;
2195 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2196 IVT, Promoted);
2197
2198 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2199 ST->getMemOperand());
2200 }
2201
2202 //===----------------------------------------------------------------------===//
2203 // Float Result Promotion
2204 //===----------------------------------------------------------------------===//
2205
2206 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2207 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2208 dbgs() << "\n");
2209 SDValue R = SDValue();
2210
2211 // See if the target wants to custom expand this node.
2212 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2213 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2214 return;
2215 }
2216
2217 switch (N->getOpcode()) {
2218 // These opcodes cannot appear if promotion of FP16 is done in the backend
2219 // instead of Clang
2220 case ISD::FP16_TO_FP:
2221 case ISD::FP_TO_FP16:
2222 default:
2223 #ifndef NDEBUG
2224 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2225 N->dump(&DAG); dbgs() << "\n";
2226 #endif
2227 llvm_unreachable("Do not know how to promote this operator's result!");
2228
2229 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2230 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2231 case ISD::EXTRACT_VECTOR_ELT:
2232 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2233 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2234
2235 // Unary FP Operations
2236 case ISD::FABS:
2237 case ISD::FCBRT:
2238 case ISD::FCEIL:
2239 case ISD::FCOS:
2240 case ISD::FEXP:
2241 case ISD::FEXP2:
2242 case ISD::FFLOOR:
2243 case ISD::FLOG:
2244 case ISD::FLOG2:
2245 case ISD::FLOG10:
2246 case ISD::FNEARBYINT:
2247 case ISD::FNEG:
2248 case ISD::FRINT:
2249 case ISD::FROUND:
2250 case ISD::FROUNDEVEN:
2251 case ISD::FSIN:
2252 case ISD::FSQRT:
2253 case ISD::FTRUNC:
2254 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2255
2256 // Binary FP Operations
2257 case ISD::FADD:
2258 case ISD::FDIV:
2259 case ISD::FMAXIMUM:
2260 case ISD::FMINIMUM:
2261 case ISD::FMAXNUM:
2262 case ISD::FMINNUM:
2263 case ISD::FMUL:
2264 case ISD::FPOW:
2265 case ISD::FREM:
2266 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2267
2268 case ISD::FMA: // FMA is same as FMAD
2269 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2270
2271 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2272
2273 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2274 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2275 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2276 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2277
2278 case ISD::SINT_TO_FP:
2279 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2280 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2281 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2282 case ISD::VECREDUCE_FADD:
2283 case ISD::VECREDUCE_FMUL:
2284 case ISD::VECREDUCE_FMIN:
2285 case ISD::VECREDUCE_FMAX:
2286 R = PromoteFloatRes_VECREDUCE(N);
2287 break;
2288 case ISD::VECREDUCE_SEQ_FADD:
2289 case ISD::VECREDUCE_SEQ_FMUL:
2290 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2291 break;
2292 }
2293
2294 if (R.getNode())
2295 SetPromotedFloat(SDValue(N, ResNo), R);
2296 }
2297
2298 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2299 // At this point, it is not possible to determine if the bitcast value is
2300 // eventually stored to memory or promoted to f32 or promoted to a floating
2301 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2302 // STORE promotion handlers.
2303 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2304 EVT VT = N->getValueType(0);
2305 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2306 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2307 // bitcast will be legalized further if necessary.
2308 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2309 N->getOperand(0).getValueType().getSizeInBits());
2310 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2311 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2312 }
2313
2314 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2315 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2316 EVT VT = N->getValueType(0);
2317 SDLoc DL(N);
2318
2319 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2320 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2321 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2322 IVT);
2323
2324 // Convert the Constant to the desired FP type
2325 // FIXME We might be able to do the conversion during compilation and get rid
2326 // of it from the object code
2327 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2328 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2329 }
2330
2331 // If the Index operand is a constant, try to redirect the extract operation to
2332 // the correct legalized vector. If not, bit-convert the input vector to
2333 // equivalent integer vector. Extract the element as an (bit-cast) integer
2334 // value and convert it to the promoted type.
2335 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2336 SDLoc DL(N);
2337
2338 // If the index is constant, try to extract the value from the legalized
2339 // vector type.
2340 if (isa<ConstantSDNode>(N->getOperand(1))) {
2341 SDValue Vec = N->getOperand(0);
2342 SDValue Idx = N->getOperand(1);
2343 EVT VecVT = Vec->getValueType(0);
2344 EVT EltVT = VecVT.getVectorElementType();
2345
2346 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2347
2348 switch (getTypeAction(VecVT)) {
2349 default: break;
2350 case TargetLowering::TypeScalarizeVector: {
2351 SDValue Res = GetScalarizedVector(N->getOperand(0));
2352 ReplaceValueWith(SDValue(N, 0), Res);
2353 return SDValue();
2354 }
2355 case TargetLowering::TypeWidenVector: {
2356 Vec = GetWidenedVector(Vec);
2357 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2358 ReplaceValueWith(SDValue(N, 0), Res);
2359 return SDValue();
2360 }
2361 case TargetLowering::TypeSplitVector: {
2362 SDValue Lo, Hi;
2363 GetSplitVector(Vec, Lo, Hi);
2364
2365 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2366 SDValue Res;
2367 if (IdxVal < LoElts)
2368 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2369 else
2370 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2371 DAG.getConstant(IdxVal - LoElts, DL,
2372 Idx.getValueType()));
2373 ReplaceValueWith(SDValue(N, 0), Res);
2374 return SDValue();
2375 }
2376
2377 }
2378 }
2379
2380 // Bit-convert the input vector to the equivalent integer vector
2381 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2382 EVT IVT = NewOp.getValueType().getVectorElementType();
2383
2384 // Extract the element as an (bit-cast) integer value
2385 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2386 NewOp, N->getOperand(1));
2387
2388 // Convert the element to the desired FP type
2389 EVT VT = N->getValueType(0);
2390 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2391 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2392 }
2393
2394 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2395 // needs promotion, so does the argument X. Note that Y, if needed, will be
2396 // handled during operand promotion.
2397 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2398 EVT VT = N->getValueType(0);
2399 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2400 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2401
2402 SDValue Op1 = N->getOperand(1);
2403
2404 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2405 }
2406
2407 // Unary operation where the result and the operand have PromoteFloat type
2408 // action. Construct a new SDNode with the promoted float value of the old
2409 // operand.
2410 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2411 EVT VT = N->getValueType(0);
2412 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2413 SDValue Op = GetPromotedFloat(N->getOperand(0));
2414
2415 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2416 }
2417
2418 // Binary operations where the result and both operands have PromoteFloat type
2419 // action. Construct a new SDNode with the promoted float values of the old
2420 // operands.
2421 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2422 EVT VT = N->getValueType(0);
2423 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2424 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2425 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2426 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2427 }
2428
2429 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2430 EVT VT = N->getValueType(0);
2431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2432 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2433 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2434 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2435
2436 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2437 }
2438
2439 // Promote the Float (first) operand and retain the Integer (second) operand
2440 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2441 EVT VT = N->getValueType(0);
2442 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2443 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2444 SDValue Op1 = N->getOperand(1);
2445
2446 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2447 }
2448
2449 // Explicit operation to reduce precision. Reduce the value to half precision
2450 // and promote it back to the legal type.
2451 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2452 SDLoc DL(N);
2453
2454 SDValue Op = N->getOperand(0);
2455 EVT VT = N->getValueType(0);
2456 EVT OpVT = Op->getValueType(0);
2457 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2458 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2459
2460 // Round promoted float to desired precision
2461 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2462 // Promote it back to the legal output type
2463 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2464 }
2465
2466 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2467 LoadSDNode *L = cast<LoadSDNode>(N);
2468 EVT VT = N->getValueType(0);
2469
2470 // Load the value as an integer value with the same number of bits.
2471 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2472 SDValue newL = DAG.getLoad(
2473 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2474 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2475 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2476 // Legalize the chain result by replacing uses of the old value chain with the
2477 // new one
2478 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2479
2480 // Convert the integer value to the desired FP type
2481 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2482 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2483 }
2484
2485 // Construct a new SELECT node with the promoted true- and false- values.
2486 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2487 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2488 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2489
2490 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2491 N->getOperand(0), TrueVal, FalseVal);
2492 }
2493
2494 // Construct a new SELECT_CC node with the promoted true- and false- values.
2495 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2496 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2497 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2498 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2499
2500 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2501 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2502 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2503 }
2504
2505 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2506 // float type.
2507 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2508 SDLoc DL(N);
2509 EVT VT = N->getValueType(0);
2510 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2511 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2512 // Round the value to the desired precision (that of the source type).
2513 return DAG.getNode(
2514 ISD::FP_EXTEND, DL, NVT,
2515 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2516 }
2517
2518 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2519 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2520 N->getValueType(0)));
2521 }
2522
2523 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2524 // Expand and promote recursively.
2525 // TODO: This is non-optimal, but dealing with the concurrently happening
2526 // vector-legalization is non-trivial. We could do something similar to
2527 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2528 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2529 return SDValue();
2530 }
2531
2532 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2533 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2534 return SDValue();
2535 }
2536
2537 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2538 EVT VT = N->getValueType(0);
2539
2540 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2541 SDLoc SL(N);
2542
2543 SDValue CastVal = BitConvertToInteger(AM->getVal());
2544 EVT CastVT = CastVal.getValueType();
2545
2546 SDValue NewAtomic
2547 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2548 DAG.getVTList(CastVT, MVT::Other),
2549 { AM->getChain(), AM->getBasePtr(), CastVal },
2550 AM->getMemOperand());
2551
2552 SDValue Result = NewAtomic;
2553
2554 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2555 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2556 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2557 NewAtomic);
2558 }
2559
2560 // Legalize the chain result by replacing uses of the old value chain with the
2561 // new one
2562 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2563
2564 return Result;
2565
2566 }
2567
2568 //===----------------------------------------------------------------------===//
2569 // Half Result Soft Promotion
2570 //===----------------------------------------------------------------------===//
2571
2572 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2573 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2574 N->dump(&DAG); dbgs() << "\n");
2575 SDValue R = SDValue();
2576
2577 // See if the target wants to custom expand this node.
2578 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2579 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2580 return;
2581 }
2582
2583 switch (N->getOpcode()) {
2584 default:
2585 #ifndef NDEBUG
2586 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2587 N->dump(&DAG); dbgs() << "\n";
2588 #endif
2589 llvm_unreachable("Do not know how to soft promote this operator's result!");
2590
2591 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2592 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2593 case ISD::EXTRACT_VECTOR_ELT:
2594 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2595 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2596 case ISD::STRICT_FP_ROUND:
2597 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2598
2599 // Unary FP Operations
2600 case ISD::FABS:
2601 case ISD::FCBRT:
2602 case ISD::FCEIL:
2603 case ISD::FCOS:
2604 case ISD::FEXP:
2605 case ISD::FEXP2:
2606 case ISD::FFLOOR:
2607 case ISD::FLOG:
2608 case ISD::FLOG2:
2609 case ISD::FLOG10:
2610 case ISD::FNEARBYINT:
2611 case ISD::FNEG:
2612 case ISD::FREEZE:
2613 case ISD::FRINT:
2614 case ISD::FROUND:
2615 case ISD::FROUNDEVEN:
2616 case ISD::FSIN:
2617 case ISD::FSQRT:
2618 case ISD::FTRUNC:
2619 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2620
2621 // Binary FP Operations
2622 case ISD::FADD:
2623 case ISD::FDIV:
2624 case ISD::FMAXIMUM:
2625 case ISD::FMINIMUM:
2626 case ISD::FMAXNUM:
2627 case ISD::FMINNUM:
2628 case ISD::FMUL:
2629 case ISD::FPOW:
2630 case ISD::FREM:
2631 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2632
2633 case ISD::FMA: // FMA is same as FMAD
2634 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2635
2636 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2637
2638 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2639 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2640 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2641 case ISD::SINT_TO_FP:
2642 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2643 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2644 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2645 case ISD::VECREDUCE_FADD:
2646 case ISD::VECREDUCE_FMUL:
2647 case ISD::VECREDUCE_FMIN:
2648 case ISD::VECREDUCE_FMAX:
2649 R = SoftPromoteHalfRes_VECREDUCE(N);
2650 break;
2651 case ISD::VECREDUCE_SEQ_FADD:
2652 case ISD::VECREDUCE_SEQ_FMUL:
2653 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2654 break;
2655 }
2656
2657 if (R.getNode())
2658 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2659 }
2660
2661 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2662 return BitConvertToInteger(N->getOperand(0));
2663 }
2664
2665 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2666 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2667
2668 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2669 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2670 MVT::i16);
2671 }
2672
2673 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2674 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2675 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2676 NewOp.getValueType().getVectorElementType(), NewOp,
2677 N->getOperand(1));
2678 }
2679
2680 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2681 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2682 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2683 SDLoc dl(N);
2684
2685 EVT LVT = LHS.getValueType();
2686 EVT RVT = RHS.getValueType();
2687
2688 unsigned LSize = LVT.getSizeInBits();
2689 unsigned RSize = RVT.getSizeInBits();
2690
2691 // First get the sign bit of second operand.
2692 SDValue SignBit = DAG.getNode(
2693 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2694 DAG.getConstant(RSize - 1, dl,
2695 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2696 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2697
2698 // Shift right or sign-extend it if the two operands have different types.
2699 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2700 if (SizeDiff > 0) {
2701 SignBit =
2702 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2703 DAG.getConstant(SizeDiff, dl,
2704 TLI.getShiftAmountTy(SignBit.getValueType(),
2705 DAG.getDataLayout())));
2706 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2707 } else if (SizeDiff < 0) {
2708 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2709 SignBit =
2710 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2711 DAG.getConstant(-SizeDiff, dl,
2712 TLI.getShiftAmountTy(SignBit.getValueType(),
2713 DAG.getDataLayout())));
2714 }
2715
2716 // Clear the sign bit of the first operand.
2717 SDValue Mask = DAG.getNode(
2718 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2719 DAG.getConstant(LSize - 1, dl,
2720 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2721 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2722 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2723
2724 // Or the value with the sign bit.
2725 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2726 }
2727
2728 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2729 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2730 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2731 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2732 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2733 SDLoc dl(N);
2734
2735 // Promote to the larger FP type.
2736 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2737 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2738 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2);
2739
2740 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2741
2742 // Convert back to FP16 as an integer.
2743 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2744 }
2745
2746 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2747 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2748 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2749 SDValue Op1 = N->getOperand(1);
2750 SDLoc dl(N);
2751
2752 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2753
2754 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2755
2756 // Convert back to FP16 as an integer.
2757 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2758 }
2759
2760 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2761 if (N->isStrictFPOpcode()) {
2762 SDValue Res =
2763 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2764 {N->getOperand(0), N->getOperand(1)});
2765 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2766 return Res;
2767 }
2768
2769 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0));
2770 }
2771
2772 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2773 LoadSDNode *L = cast<LoadSDNode>(N);
2774
2775 // Load the value as an integer value with the same number of bits.
2776 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2777 SDValue NewL =
2778 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2779 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2780 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2781 L->getMemOperand()->getFlags(), L->getAAInfo());
2782 // Legalize the chain result by replacing uses of the old value chain with the
2783 // new one
2784 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2785 return NewL;
2786 }
2787
2788 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2789 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2790 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2791 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2792 Op2);
2793 }
2794
2795 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2796 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2797 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2798 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2799 N->getOperand(0), N->getOperand(1), Op2, Op3,
2800 N->getOperand(4));
2801 }
2802
2803 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2804 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2805 SDLoc dl(N);
2806
2807 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2808
2809 // Round the value to the softened type.
2810 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2811 }
2812
2813 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2814 return DAG.getUNDEF(MVT::i16);
2815 }
2816
2817 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2818 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2819 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2820 SDLoc dl(N);
2821
2822 // Promote to the larger FP type.
2823 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2824
2825 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2826
2827 // Convert back to FP16 as an integer.
2828 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2829 }
2830
2831 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2833 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2834 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2835 SDLoc dl(N);
2836
2837 // Promote to the larger FP type.
2838 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2839 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2840
2841 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2842
2843 // Convert back to FP16 as an integer.
2844 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2845 }
2846
2847 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2848 // Expand and soften recursively.
2849 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2850 return SDValue();
2851 }
2852
2853 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2854 // Expand and soften.
2855 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2856 return SDValue();
2857 }
2858
2859 //===----------------------------------------------------------------------===//
2860 // Half Operand Soft Promotion
2861 //===----------------------------------------------------------------------===//
2862
2863 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2864 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2865 N->dump(&DAG); dbgs() << "\n");
2866 SDValue Res = SDValue();
2867
2868 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2869 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2870 return false;
2871 }
2872
2873 // Nodes that use a promotion-requiring floating point operand, but doesn't
2874 // produce a soft promotion-requiring floating point result, need to be
2875 // legalized to use the soft promoted float operand. Nodes that produce at
2876 // least one soft promotion-requiring floating point result have their
2877 // operands legalized as a part of PromoteFloatResult.
2878 switch (N->getOpcode()) {
2879 default:
2880 #ifndef NDEBUG
2881 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2882 N->dump(&DAG); dbgs() << "\n";
2883 #endif
2884 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2885
2886 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2887 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2888 case ISD::FP_TO_SINT:
2889 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2890 case ISD::FP_TO_SINT_SAT:
2891 case ISD::FP_TO_UINT_SAT:
2892 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2893 case ISD::STRICT_FP_EXTEND:
2894 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2895 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2896 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2897 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2898 }
2899
2900 if (!Res.getNode())
2901 return false;
2902
2903 assert(Res.getNode() != N && "Expected a new node!");
2904
2905 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2906 "Invalid operand expansion");
2907
2908 ReplaceValueWith(SDValue(N, 0), Res);
2909 return false;
2910 }
2911
2912 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2913 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2914
2915 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2916 }
2917
2918 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2919 unsigned OpNo) {
2920 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2921 SDValue Op1 = N->getOperand(1);
2922 SDLoc dl(N);
2923
2924 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2925
2926 Op1 = GetSoftPromotedHalf(Op1);
2927 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2928
2929 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2930 Op1);
2931 }
2932
2933 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2934 bool IsStrict = N->isStrictFPOpcode();
2935 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
2936
2937 if (IsStrict) {
2938 SDValue Res =
2939 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
2940 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
2941 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2942 ReplaceValueWith(SDValue(N, 0), Res);
2943 return SDValue();
2944 }
2945
2946 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op);
2947 }
2948
2949 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
2950 SDValue Op = N->getOperand(0);
2951 SDLoc dl(N);
2952
2953 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2954
2955 Op = GetSoftPromotedHalf(Op);
2956
2957 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2958
2959 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
2960 }
2961
2962 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
2963 SDValue Op = N->getOperand(0);
2964 SDLoc dl(N);
2965
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2967
2968 Op = GetSoftPromotedHalf(Op);
2969
2970 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2971
2972 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
2973 N->getOperand(1));
2974 }
2975
2976 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
2977 unsigned OpNo) {
2978 assert(OpNo == 0 && "Can only soften the comparison values");
2979 SDValue Op0 = N->getOperand(0);
2980 SDValue Op1 = N->getOperand(1);
2981 SDLoc dl(N);
2982
2983 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2984
2985 Op0 = GetSoftPromotedHalf(Op0);
2986 Op1 = GetSoftPromotedHalf(Op1);
2987
2988 // Promote to the larger FP type.
2989 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2990 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2991
2992 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
2993 N->getOperand(2), N->getOperand(3), N->getOperand(4));
2994 }
2995
2996 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
2997 SDValue Op0 = N->getOperand(0);
2998 SDValue Op1 = N->getOperand(1);
2999 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3000 SDLoc dl(N);
3001
3002 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3003
3004 Op0 = GetSoftPromotedHalf(Op0);
3005 Op1 = GetSoftPromotedHalf(Op1);
3006
3007 // Promote to the larger FP type.
3008 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
3009 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
3010
3011 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3012 }
3013
3014 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3015 assert(OpNo == 1 && "Can only soften the stored value!");
3016 StoreSDNode *ST = cast<StoreSDNode>(N);
3017 SDValue Val = ST->getValue();
3018 SDLoc dl(N);
3019
3020 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3021 SDValue Promoted = GetSoftPromotedHalf(Val);
3022 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3023 ST->getMemOperand());
3024 }
LLVM monorepo