?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[SelectionDAG] Fix the representation of ISD::STEP_VECTOR.
[llvm-project.git] / llvm / lib / CodeGen / SelectionDAG / LegalizeFloatTypes.cpp
blob3553f9ec16c2ee3f802cfed079b10319bbafc3f5
1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements float type expansion and softening for LegalizeTypes.
10 // Softening is the act of turning a computation in an illegal floating point
11 // type into a computation in an integer type of the same size; also known as
12 // "soft float". For example, turning f32 arithmetic into operations using i32.
13 // The resulting integer value is the same as what you would get by performing
14 // the floating point operation and bitcasting the result to the integer type.
15 // Expansion is the act of changing a computation in an illegal type to be a
16 // computation in two identical registers of a smaller type. For example,
17 // implementing ppcf128 arithmetic in two f64 registers.
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 /// GetFPLibCall - Return the right libcall for the given floating point type.
30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31 /// refactored away (see RTLIB::getPOWI for an example).
32 static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45 }
46
47 //===----------------------------------------------------------------------===//
48 // Convert Float Results to Integer
49 //===----------------------------------------------------------------------===//
50
51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58 #ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61 #endif
62 llvm_unreachable("Do not know how to soften the result of this operator!");
63
64 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
65 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
66 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
67 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
68 case ISD::EXTRACT_VECTOR_ELT:
69 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
70 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
71 case ISD::STRICT_FMINNUM:
72 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
73 case ISD::STRICT_FMAXNUM:
74 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
75 case ISD::STRICT_FADD:
76 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
77 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
78 case ISD::STRICT_FCEIL:
79 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
80 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
81 case ISD::STRICT_FCOS:
82 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
83 case ISD::STRICT_FDIV:
84 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
85 case ISD::STRICT_FEXP:
86 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
87 case ISD::STRICT_FEXP2:
88 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
89 case ISD::STRICT_FFLOOR:
90 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
91 case ISD::STRICT_FLOG:
92 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
93 case ISD::STRICT_FLOG2:
94 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
95 case ISD::STRICT_FLOG10:
96 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
97 case ISD::STRICT_FMA:
98 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
99 case ISD::STRICT_FMUL:
100 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
101 case ISD::STRICT_FNEARBYINT:
102 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
103 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
104 case ISD::STRICT_FP_EXTEND:
105 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
106 case ISD::STRICT_FP_ROUND:
107 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
108 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
109 case ISD::STRICT_FPOW:
110 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
111 case ISD::STRICT_FPOWI:
112 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
113 case ISD::STRICT_FREM:
114 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
115 case ISD::STRICT_FRINT:
116 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
117 case ISD::STRICT_FROUND:
118 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
119 case ISD::STRICT_FROUNDEVEN:
120 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
121 case ISD::STRICT_FSIN:
122 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
123 case ISD::STRICT_FSQRT:
124 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
125 case ISD::STRICT_FSUB:
126 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
127 case ISD::STRICT_FTRUNC:
128 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
129 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
130 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
131 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
132 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
133 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
134 case ISD::STRICT_SINT_TO_FP:
135 case ISD::STRICT_UINT_TO_FP:
136 case ISD::SINT_TO_FP:
137 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
138 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
139 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
140 case ISD::VECREDUCE_FADD:
141 case ISD::VECREDUCE_FMUL:
142 case ISD::VECREDUCE_FMIN:
143 case ISD::VECREDUCE_FMAX:
144 R = SoftenFloatRes_VECREDUCE(N);
145 break;
146 case ISD::VECREDUCE_SEQ_FADD:
147 case ISD::VECREDUCE_SEQ_FMUL:
148 R = SoftenFloatRes_VECREDUCE_SEQ(N);
149 break;
150 }
151
152 // If R is null, the sub-method took care of registering the result.
153 if (R.getNode()) {
154 assert(R.getNode() != N);
155 SetSoftenedFloat(SDValue(N, ResNo), R);
156 }
157 }
158
159 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
160 bool IsStrict = N->isStrictFPOpcode();
161 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
162 unsigned Offset = IsStrict ? 1 : 0;
163 assert(N->getNumOperands() == (1 + Offset) &&
164 "Unexpected number of operands!");
165 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
166 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
167 TargetLowering::MakeLibCallOptions CallOptions;
168 EVT OpVT = N->getOperand(0 + Offset).getValueType();
169 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
170 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
171 CallOptions, SDLoc(N),
172 Chain);
173 if (IsStrict)
174 ReplaceValueWith(SDValue(N, 1), Tmp.second);
175 return Tmp.first;
176 }
177
178 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
179 bool IsStrict = N->isStrictFPOpcode();
180 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
181 unsigned Offset = IsStrict ? 1 : 0;
182 assert(N->getNumOperands() == (2 + Offset) &&
183 "Unexpected number of operands!");
184 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
185 GetSoftenedFloat(N->getOperand(1 + Offset)) };
186 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
187 TargetLowering::MakeLibCallOptions CallOptions;
188 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
189 N->getOperand(1 + Offset).getValueType() };
190 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
192 CallOptions, SDLoc(N),
193 Chain);
194 if (IsStrict)
195 ReplaceValueWith(SDValue(N, 1), Tmp.second);
196 return Tmp.first;
197 }
198
199 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
200 return BitConvertToInteger(N->getOperand(0));
201 }
202
203 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
204 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
205 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
206 GetSoftenedFloat(N->getOperand(0)));
207 }
208
209 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
210 unsigned ResNo) {
211 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
212 return BitConvertToInteger(Op);
213 }
214
215 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
216 // Convert the inputs to integers, and build a new pair out of them.
217 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
218 TLI.getTypeToTransformTo(*DAG.getContext(),
219 N->getValueType(0)),
220 BitConvertToInteger(N->getOperand(0)),
221 BitConvertToInteger(N->getOperand(1)));
222 }
223
224 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
225 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
226 // In ppcf128, the high 64 bits are always first in memory regardless
227 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
228 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
229 // way. However, APInt's are serialized in an Endian-sensitive fashion,
230 // so on big-Endian targets, the two doubles are output in the wrong
231 // order. Fix this by manually flipping the order of the high 64 bits
232 // and the low 64 bits here.
233 if (DAG.getDataLayout().isBigEndian() &&
234 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
235 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
236 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
237 APInt Val(128, words);
238 return DAG.getConstant(Val, SDLoc(CN),
239 TLI.getTypeToTransformTo(*DAG.getContext(),
240 CN->getValueType(0)));
241 } else {
242 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
243 TLI.getTypeToTransformTo(*DAG.getContext(),
244 CN->getValueType(0)));
245 }
246 }
247
248 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
249 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
250 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
251 NewOp.getValueType().getVectorElementType(),
252 NewOp, N->getOperand(1));
253 }
254
255 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
257 unsigned Size = NVT.getSizeInBits();
258
259 // Mask = ~(1 << (Size-1))
260 APInt API = APInt::getAllOnesValue(Size);
261 API.clearBit(Size - 1);
262 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
263 SDValue Op = GetSoftenedFloat(N->getOperand(0));
264 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
265 }
266
267 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
268 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
269 RTLIB::FMIN_F32,
270 RTLIB::FMIN_F64,
271 RTLIB::FMIN_F80,
272 RTLIB::FMIN_F128,
273 RTLIB::FMIN_PPCF128));
274 }
275
276 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
277 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
278 RTLIB::FMAX_F32,
279 RTLIB::FMAX_F64,
280 RTLIB::FMAX_F80,
281 RTLIB::FMAX_F128,
282 RTLIB::FMAX_PPCF128));
283 }
284
285 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
286 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
287 RTLIB::ADD_F32,
288 RTLIB::ADD_F64,
289 RTLIB::ADD_F80,
290 RTLIB::ADD_F128,
291 RTLIB::ADD_PPCF128));
292 }
293
294 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
295 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
296 RTLIB::CBRT_F32,
297 RTLIB::CBRT_F64,
298 RTLIB::CBRT_F80,
299 RTLIB::CBRT_F128,
300 RTLIB::CBRT_PPCF128));
301 }
302
303 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
304 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
305 RTLIB::CEIL_F32,
306 RTLIB::CEIL_F64,
307 RTLIB::CEIL_F80,
308 RTLIB::CEIL_F128,
309 RTLIB::CEIL_PPCF128));
310 }
311
312 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
313 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
314 SDValue RHS = BitConvertToInteger(N->getOperand(1));
315 SDLoc dl(N);
316
317 EVT LVT = LHS.getValueType();
318 EVT RVT = RHS.getValueType();
319
320 unsigned LSize = LVT.getSizeInBits();
321 unsigned RSize = RVT.getSizeInBits();
322
323 // First get the sign bit of second operand.
324 SDValue SignBit = DAG.getNode(
325 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
326 DAG.getConstant(RSize - 1, dl,
327 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
328 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
329
330 // Shift right or sign-extend it if the two operands have different types.
331 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
332 if (SizeDiff > 0) {
333 SignBit =
334 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
335 DAG.getConstant(SizeDiff, dl,
336 TLI.getShiftAmountTy(SignBit.getValueType(),
337 DAG.getDataLayout())));
338 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
339 } else if (SizeDiff < 0) {
340 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
341 SignBit =
342 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
343 DAG.getConstant(-SizeDiff, dl,
344 TLI.getShiftAmountTy(SignBit.getValueType(),
345 DAG.getDataLayout())));
346 }
347
348 // Clear the sign bit of the first operand.
349 SDValue Mask = DAG.getNode(
350 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
351 DAG.getConstant(LSize - 1, dl,
352 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
353 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
354 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
355
356 // Or the value with the sign bit.
357 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
358 }
359
360 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
361 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
362 RTLIB::COS_F32,
363 RTLIB::COS_F64,
364 RTLIB::COS_F80,
365 RTLIB::COS_F128,
366 RTLIB::COS_PPCF128));
367 }
368
369 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
370 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
371 RTLIB::DIV_F32,
372 RTLIB::DIV_F64,
373 RTLIB::DIV_F80,
374 RTLIB::DIV_F128,
375 RTLIB::DIV_PPCF128));
376 }
377
378 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
379 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
380 RTLIB::EXP_F32,
381 RTLIB::EXP_F64,
382 RTLIB::EXP_F80,
383 RTLIB::EXP_F128,
384 RTLIB::EXP_PPCF128));
385 }
386
387 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
388 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
389 RTLIB::EXP2_F32,
390 RTLIB::EXP2_F64,
391 RTLIB::EXP2_F80,
392 RTLIB::EXP2_F128,
393 RTLIB::EXP2_PPCF128));
394 }
395
396 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
397 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
398 RTLIB::FLOOR_F32,
399 RTLIB::FLOOR_F64,
400 RTLIB::FLOOR_F80,
401 RTLIB::FLOOR_F128,
402 RTLIB::FLOOR_PPCF128));
403 }
404
405 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
406 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
407 RTLIB::LOG_F32,
408 RTLIB::LOG_F64,
409 RTLIB::LOG_F80,
410 RTLIB::LOG_F128,
411 RTLIB::LOG_PPCF128));
412 }
413
414 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
415 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
416 RTLIB::LOG2_F32,
417 RTLIB::LOG2_F64,
418 RTLIB::LOG2_F80,
419 RTLIB::LOG2_F128,
420 RTLIB::LOG2_PPCF128));
421 }
422
423 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
424 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
425 RTLIB::LOG10_F32,
426 RTLIB::LOG10_F64,
427 RTLIB::LOG10_F80,
428 RTLIB::LOG10_F128,
429 RTLIB::LOG10_PPCF128));
430 }
431
432 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
433 bool IsStrict = N->isStrictFPOpcode();
434 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
435 unsigned Offset = IsStrict ? 1 : 0;
436 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
437 GetSoftenedFloat(N->getOperand(1 + Offset)),
438 GetSoftenedFloat(N->getOperand(2 + Offset)) };
439 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
440 TargetLowering::MakeLibCallOptions CallOptions;
441 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
442 N->getOperand(1 + Offset).getValueType(),
443 N->getOperand(2 + Offset).getValueType() };
444 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
445 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
446 GetFPLibCall(N->getValueType(0),
447 RTLIB::FMA_F32,
448 RTLIB::FMA_F64,
449 RTLIB::FMA_F80,
450 RTLIB::FMA_F128,
451 RTLIB::FMA_PPCF128),
452 NVT, Ops, CallOptions, SDLoc(N), Chain);
453 if (IsStrict)
454 ReplaceValueWith(SDValue(N, 1), Tmp.second);
455 return Tmp.first;
456 }
457
458 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
459 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
460 RTLIB::MUL_F32,
461 RTLIB::MUL_F64,
462 RTLIB::MUL_F80,
463 RTLIB::MUL_F128,
464 RTLIB::MUL_PPCF128));
465 }
466
467 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
468 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
469 RTLIB::NEARBYINT_F32,
470 RTLIB::NEARBYINT_F64,
471 RTLIB::NEARBYINT_F80,
472 RTLIB::NEARBYINT_F128,
473 RTLIB::NEARBYINT_PPCF128));
474 }
475
476 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
477 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
478 SDLoc dl(N);
479
480 // Expand Y = FNEG(X) -> Y = X ^ sign mask
481 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
482 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
483 DAG.getConstant(SignMask, dl, NVT));
484 }
485
486 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
487 bool IsStrict = N->isStrictFPOpcode();
488 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
489 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
490
491 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
492
493 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
494 Op = GetPromotedFloat(Op);
495 // If the promotion did the FP_EXTEND to the destination type for us,
496 // there's nothing left to do here.
497 if (Op.getValueType() == N->getValueType(0))
498 return BitConvertToInteger(Op);
499 }
500
501 // There's only a libcall for f16 -> f32, so proceed in two stages. Also, it's
502 // entirely possible for both f16 and f32 to be legal, so use the fully
503 // hard-float FP_EXTEND rather than FP16_TO_FP.
504 if (Op.getValueType() == MVT::f16 && N->getValueType(0) != MVT::f32) {
505 if (IsStrict) {
506 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
507 { MVT::f32, MVT::Other }, { Chain, Op });
508 Chain = Op.getValue(1);
509 } else {
510 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
511 }
512 }
513
514 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
515 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
516 TargetLowering::MakeLibCallOptions CallOptions;
517 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
518 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
519 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
520 CallOptions, SDLoc(N),
521 Chain);
522 if (IsStrict)
523 ReplaceValueWith(SDValue(N, 1), Tmp.second);
524 return Tmp.first;
525 }
526
527 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
528 // nodes?
529 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
530 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
531 SDValue Op = N->getOperand(0);
532 TargetLowering::MakeLibCallOptions CallOptions;
533 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
534 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
535 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
536 CallOptions, SDLoc(N)).first;
537 if (N->getValueType(0) == MVT::f32)
538 return Res32;
539
540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
541 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
542 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
543 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
544 }
545
546 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
547 bool IsStrict = N->isStrictFPOpcode();
548 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
549 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
550 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
551 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
552 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
553 TargetLowering::MakeLibCallOptions CallOptions;
554 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
555 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
556 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
557 CallOptions, SDLoc(N),
558 Chain);
559 if (IsStrict)
560 ReplaceValueWith(SDValue(N, 1), Tmp.second);
561 return Tmp.first;
562 }
563
564 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
565 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
566 RTLIB::POW_F32,
567 RTLIB::POW_F64,
568 RTLIB::POW_F80,
569 RTLIB::POW_F128,
570 RTLIB::POW_PPCF128));
571 }
572
573 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
574 bool IsStrict = N->isStrictFPOpcode();
575 unsigned Offset = IsStrict ? 1 : 0;
576 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
577 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
578 "Unsupported power type!");
579 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
581 if (!TLI.getLibcallName(LC)) {
582 // Some targets don't have a powi libcall; use pow instead.
583 // FIXME: Implement this if some target needs it.
584 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
585 return DAG.getUNDEF(N->getValueType(0));
586 }
587
588 if (DAG.getLibInfo().getIntSize() !=
589 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
590 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
591 // would use the wrong type for the argument.
592 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
593 return DAG.getUNDEF(N->getValueType(0));
594 }
595
596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
597 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
598 N->getOperand(1 + Offset) };
599 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
600 TargetLowering::MakeLibCallOptions CallOptions;
601 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
602 N->getOperand(1 + Offset).getValueType() };
603 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
604 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
605 CallOptions, SDLoc(N),
606 Chain);
607 if (IsStrict)
608 ReplaceValueWith(SDValue(N, 1), Tmp.second);
609 return Tmp.first;
610 }
611
612 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
613 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
614 RTLIB::REM_F32,
615 RTLIB::REM_F64,
616 RTLIB::REM_F80,
617 RTLIB::REM_F128,
618 RTLIB::REM_PPCF128));
619 }
620
621 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
622 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
623 RTLIB::RINT_F32,
624 RTLIB::RINT_F64,
625 RTLIB::RINT_F80,
626 RTLIB::RINT_F128,
627 RTLIB::RINT_PPCF128));
628 }
629
630 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
631 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
632 RTLIB::ROUND_F32,
633 RTLIB::ROUND_F64,
634 RTLIB::ROUND_F80,
635 RTLIB::ROUND_F128,
636 RTLIB::ROUND_PPCF128));
637 }
638
639 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
640 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
641 RTLIB::ROUNDEVEN_F32,
642 RTLIB::ROUNDEVEN_F64,
643 RTLIB::ROUNDEVEN_F80,
644 RTLIB::ROUNDEVEN_F128,
645 RTLIB::ROUNDEVEN_PPCF128));
646 }
647
648 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
649 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
650 RTLIB::SIN_F32,
651 RTLIB::SIN_F64,
652 RTLIB::SIN_F80,
653 RTLIB::SIN_F128,
654 RTLIB::SIN_PPCF128));
655 }
656
657 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
658 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
659 RTLIB::SQRT_F32,
660 RTLIB::SQRT_F64,
661 RTLIB::SQRT_F80,
662 RTLIB::SQRT_F128,
663 RTLIB::SQRT_PPCF128));
664 }
665
666 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
667 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
668 RTLIB::SUB_F32,
669 RTLIB::SUB_F64,
670 RTLIB::SUB_F80,
671 RTLIB::SUB_F128,
672 RTLIB::SUB_PPCF128));
673 }
674
675 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
676 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
677 RTLIB::TRUNC_F32,
678 RTLIB::TRUNC_F64,
679 RTLIB::TRUNC_F80,
680 RTLIB::TRUNC_F128,
681 RTLIB::TRUNC_PPCF128));
682 }
683
684 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
685 LoadSDNode *L = cast<LoadSDNode>(N);
686 EVT VT = N->getValueType(0);
687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
688 SDLoc dl(N);
689
690 auto MMOFlags =
691 L->getMemOperand()->getFlags() &
692 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
693 SDValue NewL;
694 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
695 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
696 L->getChain(), L->getBasePtr(), L->getOffset(),
697 L->getPointerInfo(), NVT, L->getOriginalAlign(),
698 MMOFlags, L->getAAInfo());
699 // Legalized the chain result - switch anything that used the old chain to
700 // use the new one.
701 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
702 return NewL;
703 }
704
705 // Do a non-extending load followed by FP_EXTEND.
706 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
707 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
708 L->getPointerInfo(), L->getMemoryVT(),
709 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
710 // Legalized the chain result - switch anything that used the old chain to
711 // use the new one.
712 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
713 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
714 return BitConvertToInteger(ExtendNode);
715 }
716
717 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
718 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
719 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
720 return DAG.getSelect(SDLoc(N),
721 LHS.getValueType(), N->getOperand(0), LHS, RHS);
722 }
723
724 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
725 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
726 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
727 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
728 LHS.getValueType(), N->getOperand(0),
729 N->getOperand(1), LHS, RHS, N->getOperand(4));
730 }
731
732 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
733 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
734 N->getValueType(0)));
735 }
736
737 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
738 SDValue Chain = N->getOperand(0); // Get the chain.
739 SDValue Ptr = N->getOperand(1); // Get the pointer.
740 EVT VT = N->getValueType(0);
741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
742 SDLoc dl(N);
743
744 SDValue NewVAARG;
745 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
746 N->getConstantOperandVal(3));
747
748 // Legalized the chain result - switch anything that used the old chain to
749 // use the new one.
750 if (N != NewVAARG.getValue(1).getNode())
751 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
752 return NewVAARG;
753 }
754
755 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
756 bool IsStrict = N->isStrictFPOpcode();
757 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
758 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
759 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
760 EVT RVT = N->getValueType(0);
761 EVT NVT = EVT();
762 SDLoc dl(N);
763
764 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
765 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
766 // match. Look for an appropriate libcall.
767 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
768 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
769 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
770 NVT = (MVT::SimpleValueType)t;
771 // The source needs to big enough to hold the operand.
772 if (NVT.bitsGE(SVT))
773 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
774 }
775 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
776
777 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
778 // Sign/zero extend the argument if the libcall takes a larger type.
779 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
780 NVT, N->getOperand(IsStrict ? 1 : 0));
781 TargetLowering::MakeLibCallOptions CallOptions;
782 CallOptions.setSExt(Signed);
783 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
784 std::pair<SDValue, SDValue> Tmp =
785 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
786 Op, CallOptions, dl, Chain);
787
788 if (IsStrict)
789 ReplaceValueWith(SDValue(N, 1), Tmp.second);
790 return Tmp.first;
791 }
792
793 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
794 // Expand and soften recursively.
795 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
796 return SDValue();
797 }
798
799 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
800 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
801 return SDValue();
802 }
803
804 //===----------------------------------------------------------------------===//
805 // Convert Float Operand to Integer
806 //===----------------------------------------------------------------------===//
807
808 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
809 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
810 dbgs() << "\n");
811 SDValue Res = SDValue();
812
813 switch (N->getOpcode()) {
814 default:
815 #ifndef NDEBUG
816 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
817 N->dump(&DAG); dbgs() << "\n";
818 #endif
819 llvm_unreachable("Do not know how to soften this operator's operand!");
820
821 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
822 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
823 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
824 case ISD::STRICT_FP_ROUND:
825 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
826 case ISD::STRICT_FP_TO_SINT:
827 case ISD::STRICT_FP_TO_UINT:
828 case ISD::FP_TO_SINT:
829 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
830 case ISD::FP_TO_SINT_SAT:
831 case ISD::FP_TO_UINT_SAT:
832 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
833 case ISD::STRICT_LROUND:
834 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
835 case ISD::STRICT_LLROUND:
836 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
837 case ISD::STRICT_LRINT:
838 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
839 case ISD::STRICT_LLRINT:
840 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
841 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
842 case ISD::STRICT_FSETCC:
843 case ISD::STRICT_FSETCCS:
844 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
845 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
846 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
847 }
848
849 // If the result is null, the sub-method took care of registering results etc.
850 if (!Res.getNode()) return false;
851
852 // If the result is N, the sub-method updated N in place. Tell the legalizer
853 // core about this to re-analyze.
854 if (Res.getNode() == N)
855 return true;
856
857 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
858 "Invalid operand softening");
859
860 ReplaceValueWith(SDValue(N, 0), Res);
861 return false;
862 }
863
864 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
865 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
866
867 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
868 }
869
870 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
871 // We actually deal with the partially-softened FP_TO_FP16 node too, which
872 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
873 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
874 N->getOpcode() == ISD::STRICT_FP_ROUND);
875
876 bool IsStrict = N->isStrictFPOpcode();
877 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
878 EVT SVT = Op.getValueType();
879 EVT RVT = N->getValueType(0);
880 EVT FloatRVT = N->getOpcode() == ISD::FP_TO_FP16 ? MVT::f16 : RVT;
881
882 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
883 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
884
885 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
886 Op = GetSoftenedFloat(Op);
887 TargetLowering::MakeLibCallOptions CallOptions;
888 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
889 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
890 CallOptions, SDLoc(N),
891 Chain);
892 if (IsStrict) {
893 ReplaceValueWith(SDValue(N, 1), Tmp.second);
894 ReplaceValueWith(SDValue(N, 0), Tmp.first);
895 return SDValue();
896 }
897 return Tmp.first;
898 }
899
900 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
901 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
902 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
903
904 EVT VT = NewLHS.getValueType();
905 NewLHS = GetSoftenedFloat(NewLHS);
906 NewRHS = GetSoftenedFloat(NewRHS);
907 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
908 N->getOperand(2), N->getOperand(3));
909
910 // If softenSetCCOperands returned a scalar, we need to compare the result
911 // against zero to select between true and false values.
912 if (!NewRHS.getNode()) {
913 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
914 CCCode = ISD::SETNE;
915 }
916
917 // Update N to have the operands specified.
918 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
919 DAG.getCondCode(CCCode), NewLHS, NewRHS,
920 N->getOperand(4)),
921 0);
922 }
923
924 // Even if the result type is legal, no libcall may exactly match. (e.g. We
925 // don't have FP-i8 conversions) This helper method looks for an appropriate
926 // promoted libcall.
927 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
928 bool Signed) {
929 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
930 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
931 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
932 ++IntVT) {
933 Promoted = (MVT::SimpleValueType)IntVT;
934 // The type needs to big enough to hold the result.
935 if (Promoted.bitsGE(RetVT))
936 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
937 : RTLIB::getFPTOUINT(SrcVT, Promoted);
938 }
939 return LC;
940 }
941
942 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
943 bool IsStrict = N->isStrictFPOpcode();
944 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
945 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
946
947 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
948 EVT SVT = Op.getValueType();
949 EVT RVT = N->getValueType(0);
950 EVT NVT = EVT();
951 SDLoc dl(N);
952
953 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
954 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
955 // match, eg. we don't have fp -> i8 conversions.
956 // Look for an appropriate libcall.
957 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
958 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
959 "Unsupported FP_TO_XINT!");
960
961 Op = GetSoftenedFloat(Op);
962 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
963 TargetLowering::MakeLibCallOptions CallOptions;
964 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
965 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
966 CallOptions, dl, Chain);
967
968 // Truncate the result if the libcall returns a larger type.
969 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
970
971 if (!IsStrict)
972 return Res;
973
974 ReplaceValueWith(SDValue(N, 1), Tmp.second);
975 ReplaceValueWith(SDValue(N, 0), Res);
976 return SDValue();
977 }
978
979 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
980 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
981 return Res;
982 }
983
984 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
985 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
986 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
987
988 EVT VT = NewLHS.getValueType();
989 NewLHS = GetSoftenedFloat(NewLHS);
990 NewRHS = GetSoftenedFloat(NewRHS);
991 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
992 N->getOperand(0), N->getOperand(1));
993
994 // If softenSetCCOperands returned a scalar, we need to compare the result
995 // against zero to select between true and false values.
996 if (!NewRHS.getNode()) {
997 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
998 CCCode = ISD::SETNE;
999 }
1000
1001 // Update N to have the operands specified.
1002 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1003 N->getOperand(2), N->getOperand(3),
1004 DAG.getCondCode(CCCode)),
1005 0);
1006 }
1007
1008 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1009 bool IsStrict = N->isStrictFPOpcode();
1010 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1011 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1012 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1013 ISD::CondCode CCCode =
1014 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1015
1016 EVT VT = Op0.getValueType();
1017 SDValue NewLHS = GetSoftenedFloat(Op0);
1018 SDValue NewRHS = GetSoftenedFloat(Op1);
1019 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1020 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1021
1022 // Update N to have the operands specified.
1023 if (NewRHS.getNode()) {
1024 if (IsStrict)
1025 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1026 NewRHS, DAG.getCondCode(CCCode));
1027 else
1028 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1029 DAG.getCondCode(CCCode)), 0);
1030 }
1031
1032 // Otherwise, softenSetCCOperands returned a scalar, use it.
1033 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1034 "Unexpected setcc expansion!");
1035
1036 if (IsStrict) {
1037 ReplaceValueWith(SDValue(N, 0), NewLHS);
1038 ReplaceValueWith(SDValue(N, 1), Chain);
1039 return SDValue();
1040 }
1041 return NewLHS;
1042 }
1043
1044 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1045 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1046 assert(OpNo == 1 && "Can only soften the stored value!");
1047 StoreSDNode *ST = cast<StoreSDNode>(N);
1048 SDValue Val = ST->getValue();
1049 SDLoc dl(N);
1050
1051 if (ST->isTruncatingStore())
1052 // Do an FP_ROUND followed by a non-truncating store.
1053 Val = BitConvertToInteger(DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(),
1054 Val, DAG.getIntPtrConstant(0, dl)));
1055 else
1056 Val = GetSoftenedFloat(Val);
1057
1058 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1059 ST->getMemOperand());
1060 }
1061
1062 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1063 SDValue LHS = N->getOperand(0);
1064 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1065 SDLoc dl(N);
1066
1067 EVT LVT = LHS.getValueType();
1068 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1069 EVT RVT = RHS.getValueType();
1070
1071 unsigned LSize = LVT.getSizeInBits();
1072 unsigned RSize = RVT.getSizeInBits();
1073
1074 // Shift right or sign-extend it if the two operands have different types.
1075 int SizeDiff = RSize - LSize;
1076 if (SizeDiff > 0) {
1077 RHS =
1078 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1079 DAG.getConstant(SizeDiff, dl,
1080 TLI.getShiftAmountTy(RHS.getValueType(),
1081 DAG.getDataLayout())));
1082 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1083 } else if (SizeDiff < 0) {
1084 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1085 RHS =
1086 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1087 DAG.getConstant(-SizeDiff, dl,
1088 TLI.getShiftAmountTy(RHS.getValueType(),
1089 DAG.getDataLayout())));
1090 }
1091
1092 RHS = DAG.getBitcast(LVT, RHS);
1093 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1094 }
1095
1096 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1097 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1098 bool IsStrict = N->isStrictFPOpcode();
1099 unsigned Offset = IsStrict ? 1 : 0;
1100 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1101 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1102 TargetLowering::MakeLibCallOptions CallOptions;
1103 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1104 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1105 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1106 CallOptions, SDLoc(N),
1107 Chain);
1108 if (IsStrict) {
1109 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1110 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1111 return SDValue();
1112 }
1113
1114 return Tmp.first;
1115 }
1116
1117 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1118 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1119 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1120 RTLIB::LROUND_F32,
1121 RTLIB::LROUND_F64,
1122 RTLIB::LROUND_F80,
1123 RTLIB::LROUND_F128,
1124 RTLIB::LROUND_PPCF128));
1125 }
1126
1127 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1128 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1129 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1130 RTLIB::LLROUND_F32,
1131 RTLIB::LLROUND_F64,
1132 RTLIB::LLROUND_F80,
1133 RTLIB::LLROUND_F128,
1134 RTLIB::LLROUND_PPCF128));
1135 }
1136
1137 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1138 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1139 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1140 RTLIB::LRINT_F32,
1141 RTLIB::LRINT_F64,
1142 RTLIB::LRINT_F80,
1143 RTLIB::LRINT_F128,
1144 RTLIB::LRINT_PPCF128));
1145 }
1146
1147 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1148 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1149 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1150 RTLIB::LLRINT_F32,
1151 RTLIB::LLRINT_F64,
1152 RTLIB::LLRINT_F80,
1153 RTLIB::LLRINT_F128,
1154 RTLIB::LLRINT_PPCF128));
1155 }
1156
1157 //===----------------------------------------------------------------------===//
1158 // Float Result Expansion
1159 //===----------------------------------------------------------------------===//
1160
1161 /// ExpandFloatResult - This method is called when the specified result of the
1162 /// specified node is found to need expansion. At this point, the node may also
1163 /// have invalid operands or may have other results that need promotion, we just
1164 /// know that (at least) one result needs expansion.
1165 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1166 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1167 SDValue Lo, Hi;
1168 Lo = Hi = SDValue();
1169
1170 // See if the target wants to custom expand this node.
1171 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1172 return;
1173
1174 switch (N->getOpcode()) {
1175 default:
1176 #ifndef NDEBUG
1177 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1178 N->dump(&DAG); dbgs() << "\n";
1179 #endif
1180 llvm_unreachable("Do not know how to expand the result of this operator!");
1181
1182 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1183 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1184 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1185
1186 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1187 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1188 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1189 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1190 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1191 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1192
1193 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1194 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1195 case ISD::STRICT_FMINNUM:
1196 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1197 case ISD::STRICT_FMAXNUM:
1198 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1199 case ISD::STRICT_FADD:
1200 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1201 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1202 case ISD::STRICT_FCEIL:
1203 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1204 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1205 case ISD::STRICT_FCOS:
1206 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1207 case ISD::STRICT_FDIV:
1208 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1209 case ISD::STRICT_FEXP:
1210 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1211 case ISD::STRICT_FEXP2:
1212 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1213 case ISD::STRICT_FFLOOR:
1214 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1215 case ISD::STRICT_FLOG:
1216 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1217 case ISD::STRICT_FLOG2:
1218 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1219 case ISD::STRICT_FLOG10:
1220 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1221 case ISD::STRICT_FMA:
1222 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1223 case ISD::STRICT_FMUL:
1224 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1225 case ISD::STRICT_FNEARBYINT:
1226 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1227 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1228 case ISD::STRICT_FP_EXTEND:
1229 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1230 case ISD::STRICT_FPOW:
1231 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1232 case ISD::STRICT_FPOWI:
1233 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1234 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1235 case ISD::STRICT_FRINT:
1236 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1237 case ISD::STRICT_FROUND:
1238 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1239 case ISD::STRICT_FROUNDEVEN:
1240 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1241 case ISD::STRICT_FSIN:
1242 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1243 case ISD::STRICT_FSQRT:
1244 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1245 case ISD::STRICT_FSUB:
1246 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1247 case ISD::STRICT_FTRUNC:
1248 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1249 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1250 case ISD::STRICT_SINT_TO_FP:
1251 case ISD::STRICT_UINT_TO_FP:
1252 case ISD::SINT_TO_FP:
1253 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1254 case ISD::STRICT_FREM:
1255 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1256 }
1257
1258 // If Lo/Hi is null, the sub-method took care of registering results etc.
1259 if (Lo.getNode())
1260 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1261 }
1262
1263 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1264 SDValue &Hi) {
1265 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1266 assert(NVT.getSizeInBits() == 64 &&
1267 "Do not know how to expand this float constant!");
1268 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1269 SDLoc dl(N);
1270 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1271 APInt(64, C.getRawData()[1])),
1272 dl, NVT);
1273 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1274 APInt(64, C.getRawData()[0])),
1275 dl, NVT);
1276 }
1277
1278 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1279 SDValue &Lo, SDValue &Hi) {
1280 bool IsStrict = N->isStrictFPOpcode();
1281 unsigned Offset = IsStrict ? 1 : 0;
1282 SDValue Op = N->getOperand(0 + Offset);
1283 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1284 TargetLowering::MakeLibCallOptions CallOptions;
1285 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1286 Op, CallOptions, SDLoc(N),
1287 Chain);
1288 if (IsStrict)
1289 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1290 GetPairElements(Tmp.first, Lo, Hi);
1291 }
1292
1293 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1294 SDValue &Lo, SDValue &Hi) {
1295 bool IsStrict = N->isStrictFPOpcode();
1296 unsigned Offset = IsStrict ? 1 : 0;
1297 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1298 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1299 TargetLowering::MakeLibCallOptions CallOptions;
1300 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1301 Ops, CallOptions, SDLoc(N),
1302 Chain);
1303 if (IsStrict)
1304 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1305 GetPairElements(Tmp.first, Lo, Hi);
1306 }
1307
1308 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1309 SDValue &Hi) {
1310 assert(N->getValueType(0) == MVT::ppcf128 &&
1311 "Logic only correct for ppcf128!");
1312 SDLoc dl(N);
1313 SDValue Tmp;
1314 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1315 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1316 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1317 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1318 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1319 ISD::SETEQ);
1320 }
1321
1322 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1323 SDValue &Hi) {
1324 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1325 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1326 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1327 RTLIB::FMIN_PPCF128), Lo, Hi);
1328 }
1329
1330 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1331 SDValue &Hi) {
1332 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1333 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1334 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1335 RTLIB::FMAX_PPCF128), Lo, Hi);
1336 }
1337
1338 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1339 SDValue &Hi) {
1340 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1341 RTLIB::ADD_F32, RTLIB::ADD_F64,
1342 RTLIB::ADD_F80, RTLIB::ADD_F128,
1343 RTLIB::ADD_PPCF128), Lo, Hi);
1344 }
1345
1346 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1347 SDValue &Hi) {
1348 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1349 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1350 RTLIB::CBRT_F128,
1351 RTLIB::CBRT_PPCF128), Lo, Hi);
1352 }
1353
1354 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1355 SDValue &Lo, SDValue &Hi) {
1356 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1357 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1358 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1359 RTLIB::CEIL_PPCF128), Lo, Hi);
1360 }
1361
1362 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1363 SDValue &Lo, SDValue &Hi) {
1364 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1365 RTLIB::COPYSIGN_F32,
1366 RTLIB::COPYSIGN_F64,
1367 RTLIB::COPYSIGN_F80,
1368 RTLIB::COPYSIGN_F128,
1369 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1370 }
1371
1372 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1373 SDValue &Lo, SDValue &Hi) {
1374 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1375 RTLIB::COS_F32, RTLIB::COS_F64,
1376 RTLIB::COS_F80, RTLIB::COS_F128,
1377 RTLIB::COS_PPCF128), Lo, Hi);
1378 }
1379
1380 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1381 SDValue &Hi) {
1382 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1383 RTLIB::DIV_F32,
1384 RTLIB::DIV_F64,
1385 RTLIB::DIV_F80,
1386 RTLIB::DIV_F128,
1387 RTLIB::DIV_PPCF128), Lo, Hi);
1388 }
1389
1390 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1391 SDValue &Lo, SDValue &Hi) {
1392 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1393 RTLIB::EXP_F32, RTLIB::EXP_F64,
1394 RTLIB::EXP_F80, RTLIB::EXP_F128,
1395 RTLIB::EXP_PPCF128), Lo, Hi);
1396 }
1397
1398 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1399 SDValue &Lo, SDValue &Hi) {
1400 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1401 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1402 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1403 RTLIB::EXP2_PPCF128), Lo, Hi);
1404 }
1405
1406 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1407 SDValue &Lo, SDValue &Hi) {
1408 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1409 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1410 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1411 RTLIB::FLOOR_PPCF128), Lo, Hi);
1412 }
1413
1414 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1415 SDValue &Lo, SDValue &Hi) {
1416 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1417 RTLIB::LOG_F32, RTLIB::LOG_F64,
1418 RTLIB::LOG_F80, RTLIB::LOG_F128,
1419 RTLIB::LOG_PPCF128), Lo, Hi);
1420 }
1421
1422 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1423 SDValue &Lo, SDValue &Hi) {
1424 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1425 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1426 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1427 RTLIB::LOG2_PPCF128), Lo, Hi);
1428 }
1429
1430 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1431 SDValue &Lo, SDValue &Hi) {
1432 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1433 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1434 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1435 RTLIB::LOG10_PPCF128), Lo, Hi);
1436 }
1437
1438 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1439 SDValue &Hi) {
1440 bool IsStrict = N->isStrictFPOpcode();
1441 unsigned Offset = IsStrict ? 1 : 0;
1442 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1443 N->getOperand(2 + Offset) };
1444 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1445 TargetLowering::MakeLibCallOptions CallOptions;
1446 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1447 RTLIB::FMA_F32,
1448 RTLIB::FMA_F64,
1449 RTLIB::FMA_F80,
1450 RTLIB::FMA_F128,
1451 RTLIB::FMA_PPCF128),
1452 N->getValueType(0), Ops, CallOptions,
1453 SDLoc(N), Chain);
1454 if (IsStrict)
1455 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1456 GetPairElements(Tmp.first, Lo, Hi);
1457 }
1458
1459 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1460 SDValue &Hi) {
1461 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1462 RTLIB::MUL_F32,
1463 RTLIB::MUL_F64,
1464 RTLIB::MUL_F80,
1465 RTLIB::MUL_F128,
1466 RTLIB::MUL_PPCF128), Lo, Hi);
1467 }
1468
1469 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1470 SDValue &Lo, SDValue &Hi) {
1471 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1472 RTLIB::NEARBYINT_F32,
1473 RTLIB::NEARBYINT_F64,
1474 RTLIB::NEARBYINT_F80,
1475 RTLIB::NEARBYINT_F128,
1476 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1477 }
1478
1479 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1480 SDValue &Hi) {
1481 SDLoc dl(N);
1482 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1483 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1484 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1485 }
1486
1487 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1488 SDValue &Hi) {
1489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1490 SDLoc dl(N);
1491 bool IsStrict = N->isStrictFPOpcode();
1492
1493 SDValue Chain;
1494 if (IsStrict) {
1495 // If the expanded type is the same as the input type, just bypass the node.
1496 if (NVT == N->getOperand(1).getValueType()) {
1497 Hi = N->getOperand(1);
1498 Chain = N->getOperand(0);
1499 } else {
1500 // Other we need to extend.
1501 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1502 { N->getOperand(0), N->getOperand(1) });
1503 Chain = Hi.getValue(1);
1504 }
1505 } else {
1506 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1507 }
1508
1509 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1510 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1511
1512 if (IsStrict)
1513 ReplaceValueWith(SDValue(N, 1), Chain);
1514 }
1515
1516 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1517 SDValue &Lo, SDValue &Hi) {
1518 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1519 RTLIB::POW_F32, RTLIB::POW_F64,
1520 RTLIB::POW_F80, RTLIB::POW_F128,
1521 RTLIB::POW_PPCF128), Lo, Hi);
1522 }
1523
1524 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1525 SDValue &Lo, SDValue &Hi) {
1526 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1527 }
1528
1529 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1530 SDValue &Lo, SDValue &Hi) {
1531 assert(N->getValueType(0) == MVT::ppcf128 &&
1532 "Logic only correct for ppcf128!");
1533
1534 SDLoc dl(N);
1535 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1536 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1537 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1538 }
1539
1540 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1541 SDValue &Lo, SDValue &Hi) {
1542 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1543 RTLIB::REM_F32, RTLIB::REM_F64,
1544 RTLIB::REM_F80, RTLIB::REM_F128,
1545 RTLIB::REM_PPCF128), Lo, Hi);
1546 }
1547
1548 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1549 SDValue &Lo, SDValue &Hi) {
1550 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1551 RTLIB::RINT_F32, RTLIB::RINT_F64,
1552 RTLIB::RINT_F80, RTLIB::RINT_F128,
1553 RTLIB::RINT_PPCF128), Lo, Hi);
1554 }
1555
1556 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::ROUND_F32,
1560 RTLIB::ROUND_F64,
1561 RTLIB::ROUND_F80,
1562 RTLIB::ROUND_F128,
1563 RTLIB::ROUND_PPCF128), Lo, Hi);
1564 }
1565
1566 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1567 SDValue &Lo, SDValue &Hi) {
1568 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1569 RTLIB::ROUNDEVEN_F32,
1570 RTLIB::ROUNDEVEN_F64,
1571 RTLIB::ROUNDEVEN_F80,
1572 RTLIB::ROUNDEVEN_F128,
1573 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1574 }
1575
1576 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1577 SDValue &Lo, SDValue &Hi) {
1578 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1579 RTLIB::SIN_F32, RTLIB::SIN_F64,
1580 RTLIB::SIN_F80, RTLIB::SIN_F128,
1581 RTLIB::SIN_PPCF128), Lo, Hi);
1582 }
1583
1584 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1585 SDValue &Lo, SDValue &Hi) {
1586 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1587 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1588 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1589 RTLIB::SQRT_PPCF128), Lo, Hi);
1590 }
1591
1592 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1593 SDValue &Hi) {
1594 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1595 RTLIB::SUB_F32,
1596 RTLIB::SUB_F64,
1597 RTLIB::SUB_F80,
1598 RTLIB::SUB_F128,
1599 RTLIB::SUB_PPCF128), Lo, Hi);
1600 }
1601
1602 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1603 SDValue &Lo, SDValue &Hi) {
1604 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1605 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1606 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1607 RTLIB::TRUNC_PPCF128), Lo, Hi);
1608 }
1609
1610 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1611 SDValue &Hi) {
1612 if (ISD::isNormalLoad(N)) {
1613 ExpandRes_NormalLoad(N, Lo, Hi);
1614 return;
1615 }
1616
1617 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1618 LoadSDNode *LD = cast<LoadSDNode>(N);
1619 SDValue Chain = LD->getChain();
1620 SDValue Ptr = LD->getBasePtr();
1621 SDLoc dl(N);
1622
1623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1624 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1625 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1626
1627 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1628 LD->getMemoryVT(), LD->getMemOperand());
1629
1630 // Remember the chain.
1631 Chain = Hi.getValue(1);
1632
1633 // The low part is zero.
1634 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1635 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1636
1637 // Modified the chain - switch anything that used the old chain to use the
1638 // new one.
1639 ReplaceValueWith(SDValue(LD, 1), Chain);
1640 }
1641
1642 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1643 SDValue &Hi) {
1644 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1645 EVT VT = N->getValueType(0);
1646 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1647 bool Strict = N->isStrictFPOpcode();
1648 SDValue Src = N->getOperand(Strict ? 1 : 0);
1649 EVT SrcVT = Src.getValueType();
1650 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1651 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1652 SDLoc dl(N);
1653 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1654
1655 // TODO: Any other flags to propagate?
1656 SDNodeFlags Flags;
1657 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1658
1659 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1660 // When promoting partial word types to i32 we must honor the signedness,
1661 // though.
1662 if (SrcVT.bitsLE(MVT::i32)) {
1663 // The integer can be represented exactly in an f64.
1664 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1665 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1666 if (Strict) {
1667 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1668 {Chain, Src}, Flags);
1669 Chain = Hi.getValue(1);
1670 } else
1671 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1672 } else {
1673 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1674 if (SrcVT.bitsLE(MVT::i64)) {
1675 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1676 MVT::i64, Src);
1677 LC = RTLIB::SINTTOFP_I64_PPCF128;
1678 } else if (SrcVT.bitsLE(MVT::i128)) {
1679 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1680 LC = RTLIB::SINTTOFP_I128_PPCF128;
1681 }
1682 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1683
1684 TargetLowering::MakeLibCallOptions CallOptions;
1685 CallOptions.setSExt(true);
1686 std::pair<SDValue, SDValue> Tmp =
1687 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1688 if (Strict)
1689 Chain = Tmp.second;
1690 GetPairElements(Tmp.first, Lo, Hi);
1691 }
1692
1693 // No need to complement for unsigned 32-bit integers
1694 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1695 if (Strict)
1696 ReplaceValueWith(SDValue(N, 1), Chain);
1697
1698 return;
1699 }
1700
1701 // Unsigned - fix up the SINT_TO_FP value just calculated.
1702 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1703 // keep semantics correctness if the integer is not exactly representable
1704 // here. See ExpandLegalINT_TO_FP.
1705 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1706 SrcVT = Src.getValueType();
1707
1708 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1709 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1710 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1711 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1712 ArrayRef<uint64_t> Parts;
1713
1714 switch (SrcVT.getSimpleVT().SimpleTy) {
1715 default:
1716 llvm_unreachable("Unsupported UINT_TO_FP!");
1717 case MVT::i32:
1718 Parts = TwoE32;
1719 break;
1720 case MVT::i64:
1721 Parts = TwoE64;
1722 break;
1723 case MVT::i128:
1724 Parts = TwoE128;
1725 break;
1726 }
1727
1728 // TODO: Are there other fast-math-flags to propagate to this FADD?
1729 SDValue NewLo = DAG.getConstantFP(
1730 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1731 if (Strict) {
1732 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1733 {Chain, Hi, NewLo}, Flags);
1734 Chain = Lo.getValue(1);
1735 ReplaceValueWith(SDValue(N, 1), Chain);
1736 } else
1737 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1738 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1739 Lo, Hi, ISD::SETLT);
1740 GetPairElements(Lo, Lo, Hi);
1741 }
1742
1743
1744 //===----------------------------------------------------------------------===//
1745 // Float Operand Expansion
1746 //===----------------------------------------------------------------------===//
1747
1748 /// ExpandFloatOperand - This method is called when the specified operand of the
1749 /// specified node is found to need expansion. At this point, all of the result
1750 /// types of the node are known to be legal, but other operands of the node may
1751 /// need promotion or expansion as well as the specified one.
1752 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1753 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1754 SDValue Res = SDValue();
1755
1756 // See if the target wants to custom expand this node.
1757 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1758 return false;
1759
1760 switch (N->getOpcode()) {
1761 default:
1762 #ifndef NDEBUG
1763 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1764 N->dump(&DAG); dbgs() << "\n";
1765 #endif
1766 llvm_unreachable("Do not know how to expand this operator's operand!");
1767
1768 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1769 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1770 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1771
1772 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1773 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1774 case ISD::STRICT_FP_ROUND:
1775 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1776 case ISD::STRICT_FP_TO_SINT:
1777 case ISD::STRICT_FP_TO_UINT:
1778 case ISD::FP_TO_SINT:
1779 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1780 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1781 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1782 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1783 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1784 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1785 case ISD::STRICT_FSETCC:
1786 case ISD::STRICT_FSETCCS:
1787 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1788 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1789 OpNo); break;
1790 }
1791
1792 // If the result is null, the sub-method took care of registering results etc.
1793 if (!Res.getNode()) return false;
1794
1795 // If the result is N, the sub-method updated N in place. Tell the legalizer
1796 // core about this.
1797 if (Res.getNode() == N)
1798 return true;
1799
1800 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1801 "Invalid operand expansion");
1802
1803 ReplaceValueWith(SDValue(N, 0), Res);
1804 return false;
1805 }
1806
1807 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1808 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1809 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1810 SDValue &NewRHS,
1811 ISD::CondCode &CCCode,
1812 const SDLoc &dl, SDValue &Chain,
1813 bool IsSignaling) {
1814 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1815 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1816 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1817
1818 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1819
1820 // FIXME: This generated code sucks. We want to generate
1821 // FCMPU crN, hi1, hi2
1822 // BNE crN, L:
1823 // FCMPU crN, lo1, lo2
1824 // The following can be improved, but not that much.
1825 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1826 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1827 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1828 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1829 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1830 RHSLo, CCCode, OutputChain, IsSignaling);
1831 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1832 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1833 Tmp1 =
1834 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1835 ISD::SETUNE, OutputChain, IsSignaling);
1836 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1837 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1838 RHSHi, CCCode, OutputChain, IsSignaling);
1839 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1840 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1841 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1842 NewRHS = SDValue(); // LHS is the result, not a compare.
1843 Chain = OutputChain;
1844 }
1845
1846 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1847 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1848 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1849 SDValue Chain;
1850 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1851
1852 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1853 // against zero to select between true and false values.
1854 if (!NewRHS.getNode()) {
1855 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1856 CCCode = ISD::SETNE;
1857 }
1858
1859 // Update N to have the operands specified.
1860 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1861 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1862 N->getOperand(4)), 0);
1863 }
1864
1865 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1866 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1867 "Logic only correct for ppcf128!");
1868 SDValue Lo, Hi;
1869 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1870 // The ppcf128 value is providing only the sign; take it from the
1871 // higher-order double (which must have the larger magnitude).
1872 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1873 N->getValueType(0), N->getOperand(0), Hi);
1874 }
1875
1876 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1877 bool IsStrict = N->isStrictFPOpcode();
1878 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1879 "Logic only correct for ppcf128!");
1880 SDValue Lo, Hi;
1881 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1882
1883 if (!IsStrict)
1884 // Round it the rest of the way (e.g. to f32) if needed.
1885 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1886 N->getValueType(0), Hi, N->getOperand(1));
1887
1888 // Eliminate the node if the input float type is the same as the output float
1889 // type.
1890 if (Hi.getValueType() == N->getValueType(0)) {
1891 // Connect the output chain to the input chain, unlinking the node.
1892 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1893 ReplaceValueWith(SDValue(N, 0), Hi);
1894 return SDValue();
1895 }
1896
1897 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1898 {N->getValueType(0), MVT::Other},
1899 {N->getOperand(0), Hi, N->getOperand(2)});
1900 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1901 ReplaceValueWith(SDValue(N, 0), Expansion);
1902 return SDValue();
1903 }
1904
1905 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1906 EVT RVT = N->getValueType(0);
1907 SDLoc dl(N);
1908
1909 bool IsStrict = N->isStrictFPOpcode();
1910 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1911 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1912 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1913 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1914
1915 EVT NVT;
1916 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1917 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1918 "Unsupported FP_TO_XINT!");
1919 TargetLowering::MakeLibCallOptions CallOptions;
1920 std::pair<SDValue, SDValue> Tmp =
1921 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1922 if (!IsStrict)
1923 return Tmp.first;
1924
1925 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1926 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1927 return SDValue();
1928 }
1929
1930 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1931 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1932 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1933 SDValue Chain;
1934 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1935
1936 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1937 // against zero to select between true and false values.
1938 if (!NewRHS.getNode()) {
1939 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1940 CCCode = ISD::SETNE;
1941 }
1942
1943 // Update N to have the operands specified.
1944 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1945 N->getOperand(2), N->getOperand(3),
1946 DAG.getCondCode(CCCode)), 0);
1947 }
1948
1949 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1950 bool IsStrict = N->isStrictFPOpcode();
1951 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1952 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1953 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1954 ISD::CondCode CCCode =
1955 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1956 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
1957 N->getOpcode() == ISD::STRICT_FSETCCS);
1958
1959 // FloatExpandSetCCOperands always returned a scalar.
1960 assert(!NewRHS.getNode() && "Expect to return scalar");
1961 assert(NewLHS.getValueType() == N->getValueType(0) &&
1962 "Unexpected setcc expansion!");
1963 if (Chain) {
1964 ReplaceValueWith(SDValue(N, 0), NewLHS);
1965 ReplaceValueWith(SDValue(N, 1), Chain);
1966 return SDValue();
1967 }
1968 return NewLHS;
1969 }
1970
1971 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
1972 if (ISD::isNormalStore(N))
1973 return ExpandOp_NormalStore(N, OpNo);
1974
1975 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1976 assert(OpNo == 1 && "Can only expand the stored value so far");
1977 StoreSDNode *ST = cast<StoreSDNode>(N);
1978
1979 SDValue Chain = ST->getChain();
1980 SDValue Ptr = ST->getBasePtr();
1981
1982 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
1983 ST->getValue().getValueType());
1984 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1985 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1986 (void)NVT;
1987
1988 SDValue Lo, Hi;
1989 GetExpandedOp(ST->getValue(), Lo, Hi);
1990
1991 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
1992 ST->getMemoryVT(), ST->getMemOperand());
1993 }
1994
1995 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
1996 EVT RVT = N->getValueType(0);
1997 EVT RetVT = N->getOperand(0).getValueType();
1998 TargetLowering::MakeLibCallOptions CallOptions;
1999 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2000 RTLIB::LROUND_F32,
2001 RTLIB::LROUND_F64,
2002 RTLIB::LROUND_F80,
2003 RTLIB::LROUND_F128,
2004 RTLIB::LROUND_PPCF128),
2005 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2006 }
2007
2008 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2009 EVT RVT = N->getValueType(0);
2010 EVT RetVT = N->getOperand(0).getValueType();
2011 TargetLowering::MakeLibCallOptions CallOptions;
2012 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2013 RTLIB::LLROUND_F32,
2014 RTLIB::LLROUND_F64,
2015 RTLIB::LLROUND_F80,
2016 RTLIB::LLROUND_F128,
2017 RTLIB::LLROUND_PPCF128),
2018 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2019 }
2020
2021 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2022 EVT RVT = N->getValueType(0);
2023 EVT RetVT = N->getOperand(0).getValueType();
2024 TargetLowering::MakeLibCallOptions CallOptions;
2025 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2026 RTLIB::LRINT_F32,
2027 RTLIB::LRINT_F64,
2028 RTLIB::LRINT_F80,
2029 RTLIB::LRINT_F128,
2030 RTLIB::LRINT_PPCF128),
2031 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2032 }
2033
2034 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2035 EVT RVT = N->getValueType(0);
2036 EVT RetVT = N->getOperand(0).getValueType();
2037 TargetLowering::MakeLibCallOptions CallOptions;
2038 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2039 RTLIB::LLRINT_F32,
2040 RTLIB::LLRINT_F64,
2041 RTLIB::LLRINT_F80,
2042 RTLIB::LLRINT_F128,
2043 RTLIB::LLRINT_PPCF128),
2044 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2045 }
2046
2047 //===----------------------------------------------------------------------===//
2048 // Float Operand Promotion
2049 //===----------------------------------------------------------------------===//
2050 //
2051
2052 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2053 if (OpVT == MVT::f16) {
2054 return ISD::FP16_TO_FP;
2055 } else if (RetVT == MVT::f16) {
2056 return ISD::FP_TO_FP16;
2057 }
2058
2059 report_fatal_error("Attempt at an invalid promotion-related conversion");
2060 }
2061
2062 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2063 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2064 dbgs() << "\n");
2065 SDValue R = SDValue();
2066
2067 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2068 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2069 return false;
2070 }
2071
2072 // Nodes that use a promotion-requiring floating point operand, but doesn't
2073 // produce a promotion-requiring floating point result, need to be legalized
2074 // to use the promoted float operand. Nodes that produce at least one
2075 // promotion-requiring floating point result have their operands legalized as
2076 // a part of PromoteFloatResult.
2077 switch (N->getOpcode()) {
2078 default:
2079 #ifndef NDEBUG
2080 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2081 N->dump(&DAG); dbgs() << "\n";
2082 #endif
2083 llvm_unreachable("Do not know how to promote this operator's operand!");
2084
2085 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2086 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2087 case ISD::FP_TO_SINT:
2088 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2089 case ISD::FP_TO_SINT_SAT:
2090 case ISD::FP_TO_UINT_SAT:
2091 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2092 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2093 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2094 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2095 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2096 }
2097
2098 if (R.getNode())
2099 ReplaceValueWith(SDValue(N, 0), R);
2100 return false;
2101 }
2102
2103 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2104 SDValue Op = N->getOperand(0);
2105 EVT OpVT = Op->getValueType(0);
2106
2107 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2108 EVT PromotedVT = Promoted->getValueType(0);
2109
2110 // Convert the promoted float value to the desired IVT.
2111 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2112 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2113 IVT, Promoted);
2114 // The final result type might not be an scalar so we need a bitcast. The
2115 // bitcast will be further legalized if needed.
2116 return DAG.getBitcast(N->getValueType(0), Convert);
2117 }
2118
2119 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2120 // PromoteFloatRes_FCOPYSIGN.
2121 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2122 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2123 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2124
2125 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2126 N->getOperand(0), Op1);
2127 }
2128
2129 // Convert the promoted float value to the desired integer type
2130 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2131 SDValue Op = GetPromotedFloat(N->getOperand(0));
2132 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2133 }
2134
2135 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2136 unsigned OpNo) {
2137 SDValue Op = GetPromotedFloat(N->getOperand(0));
2138 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2139 N->getOperand(1));
2140 }
2141
2142 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2143 SDValue Op = GetPromotedFloat(N->getOperand(0));
2144 EVT VT = N->getValueType(0);
2145
2146 // Desired VT is same as promoted type. Use promoted float directly.
2147 if (VT == Op->getValueType(0))
2148 return Op;
2149
2150 // Else, extend the promoted float value to the desired VT.
2151 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2152 }
2153
2154 // Promote the float operands used for comparison. The true- and false-
2155 // operands have the same type as the result and are promoted, if needed, by
2156 // PromoteFloatRes_SELECT_CC
2157 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2158 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2159 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2160
2161 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2162 LHS, RHS, N->getOperand(2), N->getOperand(3),
2163 N->getOperand(4));
2164 }
2165
2166 // Construct a SETCC that compares the promoted values and sets the conditional
2167 // code.
2168 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2169 EVT VT = N->getValueType(0);
2170 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2171 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2172 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2173
2174 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2175
2176 }
2177
2178 // Lower the promoted Float down to the integer value of same size and construct
2179 // a STORE of the integer value.
2180 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2181 StoreSDNode *ST = cast<StoreSDNode>(N);
2182 SDValue Val = ST->getValue();
2183 SDLoc DL(N);
2184
2185 SDValue Promoted = GetPromotedFloat(Val);
2186 EVT VT = ST->getOperand(1).getValueType();
2187 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2188
2189 SDValue NewVal;
2190 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2191 IVT, Promoted);
2192
2193 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2194 ST->getMemOperand());
2195 }
2196
2197 //===----------------------------------------------------------------------===//
2198 // Float Result Promotion
2199 //===----------------------------------------------------------------------===//
2200
2201 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2202 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2203 dbgs() << "\n");
2204 SDValue R = SDValue();
2205
2206 // See if the target wants to custom expand this node.
2207 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2208 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2209 return;
2210 }
2211
2212 switch (N->getOpcode()) {
2213 // These opcodes cannot appear if promotion of FP16 is done in the backend
2214 // instead of Clang
2215 case ISD::FP16_TO_FP:
2216 case ISD::FP_TO_FP16:
2217 default:
2218 #ifndef NDEBUG
2219 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2220 N->dump(&DAG); dbgs() << "\n";
2221 #endif
2222 llvm_unreachable("Do not know how to promote this operator's result!");
2223
2224 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2225 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2226 case ISD::EXTRACT_VECTOR_ELT:
2227 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2228 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2229
2230 // Unary FP Operations
2231 case ISD::FABS:
2232 case ISD::FCBRT:
2233 case ISD::FCEIL:
2234 case ISD::FCOS:
2235 case ISD::FEXP:
2236 case ISD::FEXP2:
2237 case ISD::FFLOOR:
2238 case ISD::FLOG:
2239 case ISD::FLOG2:
2240 case ISD::FLOG10:
2241 case ISD::FNEARBYINT:
2242 case ISD::FNEG:
2243 case ISD::FRINT:
2244 case ISD::FROUND:
2245 case ISD::FROUNDEVEN:
2246 case ISD::FSIN:
2247 case ISD::FSQRT:
2248 case ISD::FTRUNC:
2249 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2250
2251 // Binary FP Operations
2252 case ISD::FADD:
2253 case ISD::FDIV:
2254 case ISD::FMAXIMUM:
2255 case ISD::FMINIMUM:
2256 case ISD::FMAXNUM:
2257 case ISD::FMINNUM:
2258 case ISD::FMUL:
2259 case ISD::FPOW:
2260 case ISD::FREM:
2261 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2262
2263 case ISD::FMA: // FMA is same as FMAD
2264 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2265
2266 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2267
2268 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2269 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2270 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2271 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2272
2273 case ISD::SINT_TO_FP:
2274 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2275 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2276 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2277 case ISD::VECREDUCE_FADD:
2278 case ISD::VECREDUCE_FMUL:
2279 case ISD::VECREDUCE_FMIN:
2280 case ISD::VECREDUCE_FMAX:
2281 R = PromoteFloatRes_VECREDUCE(N);
2282 break;
2283 case ISD::VECREDUCE_SEQ_FADD:
2284 case ISD::VECREDUCE_SEQ_FMUL:
2285 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2286 break;
2287 }
2288
2289 if (R.getNode())
2290 SetPromotedFloat(SDValue(N, ResNo), R);
2291 }
2292
2293 // Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2294 // At this point, it is not possible to determine if the bitcast value is
2295 // eventually stored to memory or promoted to f32 or promoted to a floating
2296 // point at a higher precision. Some of these cases are handled by FP_EXTEND,
2297 // STORE promotion handlers.
2298 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2299 EVT VT = N->getValueType(0);
2300 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2301 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2302 // bitcast will be legalized further if necessary.
2303 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2304 N->getOperand(0).getValueType().getSizeInBits());
2305 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2306 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2307 }
2308
2309 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2310 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2311 EVT VT = N->getValueType(0);
2312 SDLoc DL(N);
2313
2314 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2315 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2316 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2317 IVT);
2318
2319 // Convert the Constant to the desired FP type
2320 // FIXME We might be able to do the conversion during compilation and get rid
2321 // of it from the object code
2322 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2323 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2324 }
2325
2326 // If the Index operand is a constant, try to redirect the extract operation to
2327 // the correct legalized vector. If not, bit-convert the input vector to
2328 // equivalent integer vector. Extract the element as an (bit-cast) integer
2329 // value and convert it to the promoted type.
2330 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2331 SDLoc DL(N);
2332
2333 // If the index is constant, try to extract the value from the legalized
2334 // vector type.
2335 if (isa<ConstantSDNode>(N->getOperand(1))) {
2336 SDValue Vec = N->getOperand(0);
2337 SDValue Idx = N->getOperand(1);
2338 EVT VecVT = Vec->getValueType(0);
2339 EVT EltVT = VecVT.getVectorElementType();
2340
2341 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2342
2343 switch (getTypeAction(VecVT)) {
2344 default: break;
2345 case TargetLowering::TypeScalarizeVector: {
2346 SDValue Res = GetScalarizedVector(N->getOperand(0));
2347 ReplaceValueWith(SDValue(N, 0), Res);
2348 return SDValue();
2349 }
2350 case TargetLowering::TypeWidenVector: {
2351 Vec = GetWidenedVector(Vec);
2352 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2353 ReplaceValueWith(SDValue(N, 0), Res);
2354 return SDValue();
2355 }
2356 case TargetLowering::TypeSplitVector: {
2357 SDValue Lo, Hi;
2358 GetSplitVector(Vec, Lo, Hi);
2359
2360 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2361 SDValue Res;
2362 if (IdxVal < LoElts)
2363 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2364 else
2365 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2366 DAG.getConstant(IdxVal - LoElts, DL,
2367 Idx.getValueType()));
2368 ReplaceValueWith(SDValue(N, 0), Res);
2369 return SDValue();
2370 }
2371
2372 }
2373 }
2374
2375 // Bit-convert the input vector to the equivalent integer vector
2376 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2377 EVT IVT = NewOp.getValueType().getVectorElementType();
2378
2379 // Extract the element as an (bit-cast) integer value
2380 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2381 NewOp, N->getOperand(1));
2382
2383 // Convert the element to the desired FP type
2384 EVT VT = N->getValueType(0);
2385 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2386 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2387 }
2388
2389 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2390 // needs promotion, so does the argument X. Note that Y, if needed, will be
2391 // handled during operand promotion.
2392 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2393 EVT VT = N->getValueType(0);
2394 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2395 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2396
2397 SDValue Op1 = N->getOperand(1);
2398
2399 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2400 }
2401
2402 // Unary operation where the result and the operand have PromoteFloat type
2403 // action. Construct a new SDNode with the promoted float value of the old
2404 // operand.
2405 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2406 EVT VT = N->getValueType(0);
2407 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2408 SDValue Op = GetPromotedFloat(N->getOperand(0));
2409
2410 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2411 }
2412
2413 // Binary operations where the result and both operands have PromoteFloat type
2414 // action. Construct a new SDNode with the promoted float values of the old
2415 // operands.
2416 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2417 EVT VT = N->getValueType(0);
2418 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2419 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2420 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2421 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2422 }
2423
2424 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2425 EVT VT = N->getValueType(0);
2426 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2427 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2428 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2429 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2430
2431 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2432 }
2433
2434 // Promote the Float (first) operand and retain the Integer (second) operand
2435 SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2436 EVT VT = N->getValueType(0);
2437 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2438 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2439 SDValue Op1 = N->getOperand(1);
2440
2441 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2442 }
2443
2444 // Explicit operation to reduce precision. Reduce the value to half precision
2445 // and promote it back to the legal type.
2446 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2447 SDLoc DL(N);
2448
2449 SDValue Op = N->getOperand(0);
2450 EVT VT = N->getValueType(0);
2451 EVT OpVT = Op->getValueType(0);
2452 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2453 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2454
2455 // Round promoted float to desired precision
2456 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2457 // Promote it back to the legal output type
2458 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2459 }
2460
2461 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2462 LoadSDNode *L = cast<LoadSDNode>(N);
2463 EVT VT = N->getValueType(0);
2464
2465 // Load the value as an integer value with the same number of bits.
2466 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2467 SDValue newL = DAG.getLoad(
2468 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2469 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2470 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2471 // Legalize the chain result by replacing uses of the old value chain with the
2472 // new one
2473 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2474
2475 // Convert the integer value to the desired FP type
2476 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2477 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2478 }
2479
2480 // Construct a new SELECT node with the promoted true- and false- values.
2481 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2482 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2483 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2484
2485 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2486 N->getOperand(0), TrueVal, FalseVal);
2487 }
2488
2489 // Construct a new SELECT_CC node with the promoted true- and false- values.
2490 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2491 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2492 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2493 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2494
2495 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2496 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2497 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2498 }
2499
2500 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
2501 // float type.
2502 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2503 SDLoc DL(N);
2504 EVT VT = N->getValueType(0);
2505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2506 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2507 // Round the value to the desired precision (that of the source type).
2508 return DAG.getNode(
2509 ISD::FP_EXTEND, DL, NVT,
2510 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, DAG.getIntPtrConstant(0, DL)));
2511 }
2512
2513 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2514 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2515 N->getValueType(0)));
2516 }
2517
2518 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2519 // Expand and promote recursively.
2520 // TODO: This is non-optimal, but dealing with the concurrently happening
2521 // vector-legalization is non-trivial. We could do something similar to
2522 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2523 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2524 return SDValue();
2525 }
2526
2527 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2528 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2529 return SDValue();
2530 }
2531
2532 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2533 EVT VT = N->getValueType(0);
2534
2535 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2536 SDLoc SL(N);
2537
2538 SDValue CastVal = BitConvertToInteger(AM->getVal());
2539 EVT CastVT = CastVal.getValueType();
2540
2541 SDValue NewAtomic
2542 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2543 DAG.getVTList(CastVT, MVT::Other),
2544 { AM->getChain(), AM->getBasePtr(), CastVal },
2545 AM->getMemOperand());
2546
2547 SDValue Result = NewAtomic;
2548
2549 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2550 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2551 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2552 NewAtomic);
2553 }
2554
2555 // Legalize the chain result by replacing uses of the old value chain with the
2556 // new one
2557 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2558
2559 return Result;
2560
2561 }
2562
2563 //===----------------------------------------------------------------------===//
2564 // Half Result Soft Promotion
2565 //===----------------------------------------------------------------------===//
2566
2567 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2568 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2569 N->dump(&DAG); dbgs() << "\n");
2570 SDValue R = SDValue();
2571
2572 // See if the target wants to custom expand this node.
2573 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2574 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2575 return;
2576 }
2577
2578 switch (N->getOpcode()) {
2579 default:
2580 #ifndef NDEBUG
2581 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2582 N->dump(&DAG); dbgs() << "\n";
2583 #endif
2584 llvm_unreachable("Do not know how to soft promote this operator's result!");
2585
2586 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2587 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2588 case ISD::EXTRACT_VECTOR_ELT:
2589 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2590 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2591 case ISD::STRICT_FP_ROUND:
2592 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2593
2594 // Unary FP Operations
2595 case ISD::FABS:
2596 case ISD::FCBRT:
2597 case ISD::FCEIL:
2598 case ISD::FCOS:
2599 case ISD::FEXP:
2600 case ISD::FEXP2:
2601 case ISD::FFLOOR:
2602 case ISD::FLOG:
2603 case ISD::FLOG2:
2604 case ISD::FLOG10:
2605 case ISD::FNEARBYINT:
2606 case ISD::FNEG:
2607 case ISD::FREEZE:
2608 case ISD::FRINT:
2609 case ISD::FROUND:
2610 case ISD::FROUNDEVEN:
2611 case ISD::FSIN:
2612 case ISD::FSQRT:
2613 case ISD::FTRUNC:
2614 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2615
2616 // Binary FP Operations
2617 case ISD::FADD:
2618 case ISD::FDIV:
2619 case ISD::FMAXIMUM:
2620 case ISD::FMINIMUM:
2621 case ISD::FMAXNUM:
2622 case ISD::FMINNUM:
2623 case ISD::FMUL:
2624 case ISD::FPOW:
2625 case ISD::FREM:
2626 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2627
2628 case ISD::FMA: // FMA is same as FMAD
2629 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2630
2631 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2632
2633 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2634 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2635 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2636 case ISD::SINT_TO_FP:
2637 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2638 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2639 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2640 case ISD::VECREDUCE_FADD:
2641 case ISD::VECREDUCE_FMUL:
2642 case ISD::VECREDUCE_FMIN:
2643 case ISD::VECREDUCE_FMAX:
2644 R = SoftPromoteHalfRes_VECREDUCE(N);
2645 break;
2646 case ISD::VECREDUCE_SEQ_FADD:
2647 case ISD::VECREDUCE_SEQ_FMUL:
2648 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2649 break;
2650 }
2651
2652 if (R.getNode())
2653 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2654 }
2655
2656 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2657 return BitConvertToInteger(N->getOperand(0));
2658 }
2659
2660 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2661 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2662
2663 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2664 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2665 MVT::i16);
2666 }
2667
2668 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2669 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2670 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2671 NewOp.getValueType().getVectorElementType(), NewOp,
2672 N->getOperand(1));
2673 }
2674
2675 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2676 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2677 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2678 SDLoc dl(N);
2679
2680 EVT LVT = LHS.getValueType();
2681 EVT RVT = RHS.getValueType();
2682
2683 unsigned LSize = LVT.getSizeInBits();
2684 unsigned RSize = RVT.getSizeInBits();
2685
2686 // First get the sign bit of second operand.
2687 SDValue SignBit = DAG.getNode(
2688 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2689 DAG.getConstant(RSize - 1, dl,
2690 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2691 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2692
2693 // Shift right or sign-extend it if the two operands have different types.
2694 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2695 if (SizeDiff > 0) {
2696 SignBit =
2697 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2698 DAG.getConstant(SizeDiff, dl,
2699 TLI.getShiftAmountTy(SignBit.getValueType(),
2700 DAG.getDataLayout())));
2701 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2702 } else if (SizeDiff < 0) {
2703 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2704 SignBit =
2705 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2706 DAG.getConstant(-SizeDiff, dl,
2707 TLI.getShiftAmountTy(SignBit.getValueType(),
2708 DAG.getDataLayout())));
2709 }
2710
2711 // Clear the sign bit of the first operand.
2712 SDValue Mask = DAG.getNode(
2713 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2714 DAG.getConstant(LSize - 1, dl,
2715 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2716 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2717 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2718
2719 // Or the value with the sign bit.
2720 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2721 }
2722
2723 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2725 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2726 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2727 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2728 SDLoc dl(N);
2729
2730 // Promote to the larger FP type.
2731 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2732 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2733 Op2 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op2);
2734
2735 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2736
2737 // Convert back to FP16 as an integer.
2738 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2739 }
2740
2741 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2742 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2743 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2744 SDValue Op1 = N->getOperand(1);
2745 SDLoc dl(N);
2746
2747 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2748
2749 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2750
2751 // Convert back to FP16 as an integer.
2752 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2753 }
2754
2755 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2756 if (N->isStrictFPOpcode()) {
2757 SDValue Res =
2758 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2759 {N->getOperand(0), N->getOperand(1)});
2760 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2761 return Res;
2762 }
2763
2764 return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0));
2765 }
2766
2767 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2768 LoadSDNode *L = cast<LoadSDNode>(N);
2769
2770 // Load the value as an integer value with the same number of bits.
2771 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2772 SDValue NewL =
2773 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2774 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2775 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2776 L->getMemOperand()->getFlags(), L->getAAInfo());
2777 // Legalize the chain result by replacing uses of the old value chain with the
2778 // new one
2779 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2780 return NewL;
2781 }
2782
2783 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2784 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2785 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2786 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2787 Op2);
2788 }
2789
2790 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2791 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2792 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2793 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2794 N->getOperand(0), N->getOperand(1), Op2, Op3,
2795 N->getOperand(4));
2796 }
2797
2798 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2799 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2800 SDLoc dl(N);
2801
2802 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2803
2804 // Round the value to the softened type.
2805 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2806 }
2807
2808 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2809 return DAG.getUNDEF(MVT::i16);
2810 }
2811
2812 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2813 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2814 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2815 SDLoc dl(N);
2816
2817 // Promote to the larger FP type.
2818 Op = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2819
2820 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2821
2822 // Convert back to FP16 as an integer.
2823 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2824 }
2825
2826 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2827 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2828 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2829 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2830 SDLoc dl(N);
2831
2832 // Promote to the larger FP type.
2833 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2834 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2835
2836 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2837
2838 // Convert back to FP16 as an integer.
2839 return DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, Res);
2840 }
2841
2842 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2843 // Expand and soften recursively.
2844 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2845 return SDValue();
2846 }
2847
2848 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2849 // Expand and soften.
2850 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2851 return SDValue();
2852 }
2853
2854 //===----------------------------------------------------------------------===//
2855 // Half Operand Soft Promotion
2856 //===----------------------------------------------------------------------===//
2857
2858 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2859 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2860 N->dump(&DAG); dbgs() << "\n");
2861 SDValue Res = SDValue();
2862
2863 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2864 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2865 return false;
2866 }
2867
2868 // Nodes that use a promotion-requiring floating point operand, but doesn't
2869 // produce a soft promotion-requiring floating point result, need to be
2870 // legalized to use the soft promoted float operand. Nodes that produce at
2871 // least one soft promotion-requiring floating point result have their
2872 // operands legalized as a part of PromoteFloatResult.
2873 switch (N->getOpcode()) {
2874 default:
2875 #ifndef NDEBUG
2876 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2877 N->dump(&DAG); dbgs() << "\n";
2878 #endif
2879 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2880
2881 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2882 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2883 case ISD::FP_TO_SINT:
2884 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2885 case ISD::FP_TO_SINT_SAT:
2886 case ISD::FP_TO_UINT_SAT:
2887 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2888 case ISD::STRICT_FP_EXTEND:
2889 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2890 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2891 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2892 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2893 }
2894
2895 if (!Res.getNode())
2896 return false;
2897
2898 assert(Res.getNode() != N && "Expected a new node!");
2899
2900 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2901 "Invalid operand expansion");
2902
2903 ReplaceValueWith(SDValue(N, 0), Res);
2904 return false;
2905 }
2906
2907 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2908 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2909
2910 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2911 }
2912
2913 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2914 unsigned OpNo) {
2915 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2916 SDValue Op1 = N->getOperand(1);
2917 SDLoc dl(N);
2918
2919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2920
2921 Op1 = GetSoftPromotedHalf(Op1);
2922 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2923
2924 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2925 Op1);
2926 }
2927
2928 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2929 bool IsStrict = N->isStrictFPOpcode();
2930 SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
2931
2932 if (IsStrict) {
2933 SDValue Res =
2934 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
2935 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
2936 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2937 ReplaceValueWith(SDValue(N, 0), Res);
2938 return SDValue();
2939 }
2940
2941 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op);
2942 }
2943
2944 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
2945 SDValue Op = N->getOperand(0);
2946 SDLoc dl(N);
2947
2948 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2949
2950 Op = GetSoftPromotedHalf(Op);
2951
2952 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2953
2954 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
2955 }
2956
2957 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
2958 SDValue Op = N->getOperand(0);
2959 SDLoc dl(N);
2960
2961 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
2962
2963 Op = GetSoftPromotedHalf(Op);
2964
2965 SDValue Res = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op);
2966
2967 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
2968 N->getOperand(1));
2969 }
2970
2971 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
2972 unsigned OpNo) {
2973 assert(OpNo == 0 && "Can only soften the comparison values");
2974 SDValue Op0 = N->getOperand(0);
2975 SDValue Op1 = N->getOperand(1);
2976 SDLoc dl(N);
2977
2978 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2979
2980 Op0 = GetSoftPromotedHalf(Op0);
2981 Op1 = GetSoftPromotedHalf(Op1);
2982
2983 // Promote to the larger FP type.
2984 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
2985 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
2986
2987 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
2988 N->getOperand(2), N->getOperand(3), N->getOperand(4));
2989 }
2990
2991 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
2992 SDValue Op0 = N->getOperand(0);
2993 SDValue Op1 = N->getOperand(1);
2994 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2995 SDLoc dl(N);
2996
2997 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
2998
2999 Op0 = GetSoftPromotedHalf(Op0);
3000 Op1 = GetSoftPromotedHalf(Op1);
3001
3002 // Promote to the larger FP type.
3003 Op0 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op0);
3004 Op1 = DAG.getNode(ISD::FP16_TO_FP, dl, NVT, Op1);
3005
3006 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3007 }
3008
3009 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3010 assert(OpNo == 1 && "Can only soften the stored value!");
3011 StoreSDNode *ST = cast<StoreSDNode>(N);
3012 SDValue Val = ST->getValue();
3013 SDLoc dl(N);
3014
3015 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3016 SDValue Promoted = GetSoftPromotedHalf(Val);
3017 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3018 ST->getMemOperand());
3019 }
LLVM monorepo