1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the SelectionDAG::Legalize method.
11 //===----------------------------------------------------------------------===//
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/CodeGen/ISDOpcodes.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineJumpTableInfo.h"
23 #include "llvm/CodeGen/MachineMemOperand.h"
24 #include "llvm/CodeGen/RuntimeLibcalls.h"
25 #include "llvm/CodeGen/SelectionDAG.h"
26 #include "llvm/CodeGen/SelectionDAGNodes.h"
27 #include "llvm/CodeGen/TargetFrameLowering.h"
28 #include "llvm/CodeGen/TargetLowering.h"
29 #include "llvm/CodeGen/TargetSubtargetInfo.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/CallingConv.h"
32 #include "llvm/IR/Constants.h"
33 #include "llvm/IR/DataLayout.h"
34 #include "llvm/IR/DerivedTypes.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/Metadata.h"
37 #include "llvm/IR/Type.h"
38 #include "llvm/Support/Casting.h"
39 #include "llvm/Support/Compiler.h"
40 #include "llvm/Support/Debug.h"
41 #include "llvm/Support/ErrorHandling.h"
42 #include "llvm/Support/MachineValueType.h"
43 #include "llvm/Support/MathExtras.h"
44 #include "llvm/Support/raw_ostream.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
55 #define DEBUG_TYPE "legalizedag"
59 /// Keeps track of state when getting the sign of a floating-point value as an
61 struct FloatSignAsInt
{
66 MachinePointerInfo IntPointerInfo
;
67 MachinePointerInfo FloatPointerInfo
;
73 //===----------------------------------------------------------------------===//
74 /// This takes an arbitrary SelectionDAG as input and
75 /// hacks on it until the target machine can handle it. This involves
76 /// eliminating value sizes the machine cannot handle (promoting small sizes to
77 /// large sizes or splitting up large values into small values) as well as
78 /// eliminating operations the machine cannot handle.
80 /// This code also does a small amount of optimization and recognition of idioms
81 /// as part of its processing. For example, if a target does not support a
82 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
83 /// will attempt merge setcc and brc instructions into brcc's.
84 class SelectionDAGLegalize
{
85 const TargetMachine
&TM
;
86 const TargetLowering
&TLI
;
89 /// The set of nodes which have already been legalized. We hold a
90 /// reference to it in order to update as necessary on node deletion.
91 SmallPtrSetImpl
<SDNode
*> &LegalizedNodes
;
93 /// A set of all the nodes updated during legalization.
94 SmallSetVector
<SDNode
*, 16> *UpdatedNodes
;
96 EVT
getSetCCResultType(EVT VT
) const {
97 return TLI
.getSetCCResultType(DAG
.getDataLayout(), *DAG
.getContext(), VT
);
100 // Libcall insertion helpers.
103 SelectionDAGLegalize(SelectionDAG
&DAG
,
104 SmallPtrSetImpl
<SDNode
*> &LegalizedNodes
,
105 SmallSetVector
<SDNode
*, 16> *UpdatedNodes
= nullptr)
106 : TM(DAG
.getTarget()), TLI(DAG
.getTargetLoweringInfo()), DAG(DAG
),
107 LegalizedNodes(LegalizedNodes
), UpdatedNodes(UpdatedNodes
) {}
109 /// Legalizes the given operation.
110 void LegalizeOp(SDNode
*Node
);
113 SDValue
OptimizeFloatStore(StoreSDNode
*ST
);
115 void LegalizeLoadOps(SDNode
*Node
);
116 void LegalizeStoreOps(SDNode
*Node
);
118 /// Some targets cannot handle a variable
119 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
120 /// is necessary to spill the vector being inserted into to memory, perform
121 /// the insert there, and then read the result back.
122 SDValue
PerformInsertVectorEltInMemory(SDValue Vec
, SDValue Val
, SDValue Idx
,
124 SDValue
ExpandINSERT_VECTOR_ELT(SDValue Vec
, SDValue Val
, SDValue Idx
,
127 /// Return a vector shuffle operation which
128 /// performs the same shuffe in terms of order or result bytes, but on a type
129 /// whose vector element type is narrower than the original shuffle type.
130 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
131 SDValue
ShuffleWithNarrowerEltType(EVT NVT
, EVT VT
, const SDLoc
&dl
,
132 SDValue N1
, SDValue N2
,
133 ArrayRef
<int> Mask
) const;
135 bool LegalizeSetCCCondCode(EVT VT
, SDValue
&LHS
, SDValue
&RHS
, SDValue
&CC
,
136 bool &NeedInvert
, const SDLoc
&dl
);
138 SDValue
ExpandLibCall(RTLIB::Libcall LC
, SDNode
*Node
, bool isSigned
);
140 std::pair
<SDValue
, SDValue
> ExpandChainLibCall(RTLIB::Libcall LC
,
141 SDNode
*Node
, bool isSigned
);
142 SDValue
ExpandFPLibCall(SDNode
*Node
, RTLIB::Libcall Call_F32
,
143 RTLIB::Libcall Call_F64
, RTLIB::Libcall Call_F80
,
144 RTLIB::Libcall Call_F128
,
145 RTLIB::Libcall Call_PPCF128
);
146 SDValue
ExpandIntLibCall(SDNode
*Node
, bool isSigned
,
147 RTLIB::Libcall Call_I8
,
148 RTLIB::Libcall Call_I16
,
149 RTLIB::Libcall Call_I32
,
150 RTLIB::Libcall Call_I64
,
151 RTLIB::Libcall Call_I128
);
152 SDValue
ExpandArgFPLibCall(SDNode
*Node
,
153 RTLIB::Libcall Call_F32
, RTLIB::Libcall Call_F64
,
154 RTLIB::Libcall Call_F80
, RTLIB::Libcall Call_F128
,
155 RTLIB::Libcall Call_PPCF128
);
156 void ExpandDivRemLibCall(SDNode
*Node
, SmallVectorImpl
<SDValue
> &Results
);
157 void ExpandSinCosLibCall(SDNode
*Node
, SmallVectorImpl
<SDValue
> &Results
);
159 SDValue
EmitStackConvert(SDValue SrcOp
, EVT SlotVT
, EVT DestVT
,
161 SDValue
EmitStackConvert(SDValue SrcOp
, EVT SlotVT
, EVT DestVT
,
162 const SDLoc
&dl
, SDValue ChainIn
);
163 SDValue
ExpandBUILD_VECTOR(SDNode
*Node
);
164 SDValue
ExpandSCALAR_TO_VECTOR(SDNode
*Node
);
165 void ExpandDYNAMIC_STACKALLOC(SDNode
*Node
,
166 SmallVectorImpl
<SDValue
> &Results
);
167 void getSignAsIntValue(FloatSignAsInt
&State
, const SDLoc
&DL
,
168 SDValue Value
) const;
169 SDValue
modifySignAsInt(const FloatSignAsInt
&State
, const SDLoc
&DL
,
170 SDValue NewIntValue
) const;
171 SDValue
ExpandFCOPYSIGN(SDNode
*Node
) const;
172 SDValue
ExpandFABS(SDNode
*Node
) const;
173 SDValue
ExpandLegalINT_TO_FP(bool isSigned
, SDValue Op0
, EVT DestVT
,
175 SDValue
PromoteLegalINT_TO_FP(SDValue LegalOp
, EVT DestVT
, bool isSigned
,
177 SDValue
PromoteLegalFP_TO_INT(SDValue LegalOp
, EVT DestVT
, bool isSigned
,
180 SDValue
ExpandBITREVERSE(SDValue Op
, const SDLoc
&dl
);
181 SDValue
ExpandBSWAP(SDValue Op
, const SDLoc
&dl
);
183 SDValue
ExpandExtractFromVectorThroughStack(SDValue Op
);
184 SDValue
ExpandInsertToVectorThroughStack(SDValue Op
);
185 SDValue
ExpandVectorBuildThroughStack(SDNode
* Node
);
187 SDValue
ExpandConstantFP(ConstantFPSDNode
*CFP
, bool UseCP
);
188 SDValue
ExpandConstant(ConstantSDNode
*CP
);
190 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
191 bool ExpandNode(SDNode
*Node
);
192 void ConvertNodeToLibcall(SDNode
*Node
);
193 void PromoteNode(SDNode
*Node
);
196 // Node replacement helpers
198 void ReplacedNode(SDNode
*N
) {
199 LegalizedNodes
.erase(N
);
201 UpdatedNodes
->insert(N
);
204 void ReplaceNode(SDNode
*Old
, SDNode
*New
) {
205 LLVM_DEBUG(dbgs() << " ... replacing: "; Old
->dump(&DAG
);
206 dbgs() << " with: "; New
->dump(&DAG
));
208 assert(Old
->getNumValues() == New
->getNumValues() &&
209 "Replacing one node with another that produces a different number "
211 DAG
.ReplaceAllUsesWith(Old
, New
);
213 UpdatedNodes
->insert(New
);
217 void ReplaceNode(SDValue Old
, SDValue New
) {
218 LLVM_DEBUG(dbgs() << " ... replacing: "; Old
->dump(&DAG
);
219 dbgs() << " with: "; New
->dump(&DAG
));
221 DAG
.ReplaceAllUsesWith(Old
, New
);
223 UpdatedNodes
->insert(New
.getNode());
224 ReplacedNode(Old
.getNode());
227 void ReplaceNode(SDNode
*Old
, const SDValue
*New
) {
228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old
->dump(&DAG
));
230 DAG
.ReplaceAllUsesWith(Old
, New
);
231 for (unsigned i
= 0, e
= Old
->getNumValues(); i
!= e
; ++i
) {
232 LLVM_DEBUG(dbgs() << (i
== 0 ? " with: " : " and: ");
235 UpdatedNodes
->insert(New
[i
].getNode());
240 void ReplaceNodeWithValue(SDValue Old
, SDValue New
) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old
->dump(&DAG
);
242 dbgs() << " with: "; New
->dump(&DAG
));
244 DAG
.ReplaceAllUsesOfValueWith(Old
, New
);
246 UpdatedNodes
->insert(New
.getNode());
247 ReplacedNode(Old
.getNode());
251 } // end anonymous namespace
253 /// Return a vector shuffle operation which
254 /// performs the same shuffle in terms of order or result bytes, but on a type
255 /// whose vector element type is narrower than the original shuffle type.
256 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
257 SDValue
SelectionDAGLegalize::ShuffleWithNarrowerEltType(
258 EVT NVT
, EVT VT
, const SDLoc
&dl
, SDValue N1
, SDValue N2
,
259 ArrayRef
<int> Mask
) const {
260 unsigned NumMaskElts
= VT
.getVectorNumElements();
261 unsigned NumDestElts
= NVT
.getVectorNumElements();
262 unsigned NumEltsGrowth
= NumDestElts
/ NumMaskElts
;
264 assert(NumEltsGrowth
&& "Cannot promote to vector type with fewer elts!");
266 if (NumEltsGrowth
== 1)
267 return DAG
.getVectorShuffle(NVT
, dl
, N1
, N2
, Mask
);
269 SmallVector
<int, 8> NewMask
;
270 for (unsigned i
= 0; i
!= NumMaskElts
; ++i
) {
272 for (unsigned j
= 0; j
!= NumEltsGrowth
; ++j
) {
274 NewMask
.push_back(-1);
276 NewMask
.push_back(Idx
* NumEltsGrowth
+ j
);
279 assert(NewMask
.size() == NumDestElts
&& "Non-integer NumEltsGrowth?");
280 assert(TLI
.isShuffleMaskLegal(NewMask
, NVT
) && "Shuffle not legal?");
281 return DAG
.getVectorShuffle(NVT
, dl
, N1
, N2
, NewMask
);
284 /// Expands the ConstantFP node to an integer constant or
285 /// a load from the constant pool.
287 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode
*CFP
, bool UseCP
) {
291 // If a FP immediate is precise when represented as a float and if the
292 // target can do an extending load from float to double, we put it into
293 // the constant pool as a float, even if it's is statically typed as a
294 // double. This shrinks FP constants and canonicalizes them for targets where
295 // an FP extending load is the same cost as a normal load (such as on the x87
296 // fp stack or PPC FP unit).
297 EVT VT
= CFP
->getValueType(0);
298 ConstantFP
*LLVMC
= const_cast<ConstantFP
*>(CFP
->getConstantFPValue());
300 assert((VT
== MVT::f64
|| VT
== MVT::f32
) && "Invalid type expansion");
301 return DAG
.getConstant(LLVMC
->getValueAPF().bitcastToAPInt(), dl
,
302 (VT
== MVT::f64
) ? MVT::i64
: MVT::i32
);
305 APFloat APF
= CFP
->getValueAPF();
309 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
310 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
311 if (!APF
.isSignaling()) {
312 while (SVT
!= MVT::f32
&& SVT
!= MVT::f16
) {
313 SVT
= (MVT::SimpleValueType
)(SVT
.getSimpleVT().SimpleTy
- 1);
314 if (ConstantFPSDNode::isValueValidForType(SVT
, APF
) &&
315 // Only do this if the target has a native EXTLOAD instruction from
317 TLI
.isLoadExtLegal(ISD::EXTLOAD
, OrigVT
, SVT
) &&
318 TLI
.ShouldShrinkFPConstant(OrigVT
)) {
319 Type
*SType
= SVT
.getTypeForEVT(*DAG
.getContext());
320 LLVMC
= cast
<ConstantFP
>(ConstantExpr::getFPTrunc(LLVMC
, SType
));
328 DAG
.getConstantPool(LLVMC
, TLI
.getPointerTy(DAG
.getDataLayout()));
329 unsigned Alignment
= cast
<ConstantPoolSDNode
>(CPIdx
)->getAlignment();
331 SDValue Result
= DAG
.getExtLoad(
332 ISD::EXTLOAD
, dl
, OrigVT
, DAG
.getEntryNode(), CPIdx
,
333 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()), VT
,
337 SDValue Result
= DAG
.getLoad(
338 OrigVT
, dl
, DAG
.getEntryNode(), CPIdx
,
339 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()), Alignment
);
343 /// Expands the Constant node to a load from the constant pool.
344 SDValue
SelectionDAGLegalize::ExpandConstant(ConstantSDNode
*CP
) {
346 EVT VT
= CP
->getValueType(0);
347 SDValue CPIdx
= DAG
.getConstantPool(CP
->getConstantIntValue(),
348 TLI
.getPointerTy(DAG
.getDataLayout()));
349 unsigned Alignment
= cast
<ConstantPoolSDNode
>(CPIdx
)->getAlignment();
350 SDValue Result
= DAG
.getLoad(
351 VT
, dl
, DAG
.getEntryNode(), CPIdx
,
352 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()), Alignment
);
356 /// Some target cannot handle a variable insertion index for the
357 /// INSERT_VECTOR_ELT instruction. In this case, it
358 /// is necessary to spill the vector being inserted into to memory, perform
359 /// the insert there, and then read the result back.
360 SDValue
SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec
,
368 // If the target doesn't support this, we have to spill the input vector
369 // to a temporary stack slot, update the element, then reload it. This is
370 // badness. We could also load the value into a vector register (either
371 // with a "move to register" or "extload into register" instruction, then
372 // permute it into place, if the idx is a constant and if the idx is
373 // supported by the target.
374 EVT VT
= Tmp1
.getValueType();
375 EVT EltVT
= VT
.getVectorElementType();
376 SDValue StackPtr
= DAG
.CreateStackTemporary(VT
);
378 int SPFI
= cast
<FrameIndexSDNode
>(StackPtr
.getNode())->getIndex();
381 SDValue Ch
= DAG
.getStore(
382 DAG
.getEntryNode(), dl
, Tmp1
, StackPtr
,
383 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), SPFI
));
385 SDValue StackPtr2
= TLI
.getVectorElementPointer(DAG
, StackPtr
, VT
, Tmp3
);
387 // Store the scalar value.
388 Ch
= DAG
.getTruncStore(Ch
, dl
, Tmp2
, StackPtr2
, MachinePointerInfo(), EltVT
);
389 // Load the updated vector.
390 return DAG
.getLoad(VT
, dl
, Ch
, StackPtr
, MachinePointerInfo::getFixedStack(
391 DAG
.getMachineFunction(), SPFI
));
394 SDValue
SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec
, SDValue Val
,
397 if (ConstantSDNode
*InsertPos
= dyn_cast
<ConstantSDNode
>(Idx
)) {
398 // SCALAR_TO_VECTOR requires that the type of the value being inserted
399 // match the element type of the vector being created, except for
400 // integers in which case the inserted value can be over width.
401 EVT EltVT
= Vec
.getValueType().getVectorElementType();
402 if (Val
.getValueType() == EltVT
||
403 (EltVT
.isInteger() && Val
.getValueType().bitsGE(EltVT
))) {
404 SDValue ScVec
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
,
405 Vec
.getValueType(), Val
);
407 unsigned NumElts
= Vec
.getValueType().getVectorNumElements();
408 // We generate a shuffle of InVec and ScVec, so the shuffle mask
409 // should be 0,1,2,3,4,5... with the appropriate element replaced with
411 SmallVector
<int, 8> ShufOps
;
412 for (unsigned i
= 0; i
!= NumElts
; ++i
)
413 ShufOps
.push_back(i
!= InsertPos
->getZExtValue() ? i
: NumElts
);
415 return DAG
.getVectorShuffle(Vec
.getValueType(), dl
, Vec
, ScVec
, ShufOps
);
418 return PerformInsertVectorEltInMemory(Vec
, Val
, Idx
, dl
);
421 SDValue
SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode
* ST
) {
422 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
423 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
424 // FIXME: We shouldn't do this for TargetConstantFP's.
425 // FIXME: move this to the DAG Combiner! Note that we can't regress due
426 // to phase ordering between legalized code and the dag combiner. This
427 // probably means that we need to integrate dag combiner and legalizer
429 // We generally can't do this one for long doubles.
430 SDValue Chain
= ST
->getChain();
431 SDValue Ptr
= ST
->getBasePtr();
432 unsigned Alignment
= ST
->getAlignment();
433 MachineMemOperand::Flags MMOFlags
= ST
->getMemOperand()->getFlags();
434 AAMDNodes AAInfo
= ST
->getAAInfo();
436 if (ConstantFPSDNode
*CFP
= dyn_cast
<ConstantFPSDNode
>(ST
->getValue())) {
437 if (CFP
->getValueType(0) == MVT::f32
&&
438 TLI
.isTypeLegal(MVT::i32
)) {
439 SDValue Con
= DAG
.getConstant(CFP
->getValueAPF().
440 bitcastToAPInt().zextOrTrunc(32),
441 SDLoc(CFP
), MVT::i32
);
442 return DAG
.getStore(Chain
, dl
, Con
, Ptr
, ST
->getPointerInfo(), Alignment
,
446 if (CFP
->getValueType(0) == MVT::f64
) {
447 // If this target supports 64-bit registers, do a single 64-bit store.
448 if (TLI
.isTypeLegal(MVT::i64
)) {
449 SDValue Con
= DAG
.getConstant(CFP
->getValueAPF().bitcastToAPInt().
450 zextOrTrunc(64), SDLoc(CFP
), MVT::i64
);
451 return DAG
.getStore(Chain
, dl
, Con
, Ptr
, ST
->getPointerInfo(),
452 Alignment
, MMOFlags
, AAInfo
);
455 if (TLI
.isTypeLegal(MVT::i32
) && !ST
->isVolatile()) {
456 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
457 // stores. If the target supports neither 32- nor 64-bits, this
458 // xform is certainly not worth it.
459 const APInt
&IntVal
= CFP
->getValueAPF().bitcastToAPInt();
460 SDValue Lo
= DAG
.getConstant(IntVal
.trunc(32), dl
, MVT::i32
);
461 SDValue Hi
= DAG
.getConstant(IntVal
.lshr(32).trunc(32), dl
, MVT::i32
);
462 if (DAG
.getDataLayout().isBigEndian())
465 Lo
= DAG
.getStore(Chain
, dl
, Lo
, Ptr
, ST
->getPointerInfo(), Alignment
,
467 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
468 DAG
.getConstant(4, dl
, Ptr
.getValueType()));
469 Hi
= DAG
.getStore(Chain
, dl
, Hi
, Ptr
,
470 ST
->getPointerInfo().getWithOffset(4),
471 MinAlign(Alignment
, 4U), MMOFlags
, AAInfo
);
473 return DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
, Hi
);
477 return SDValue(nullptr, 0);
480 void SelectionDAGLegalize::LegalizeStoreOps(SDNode
*Node
) {
481 StoreSDNode
*ST
= cast
<StoreSDNode
>(Node
);
482 SDValue Chain
= ST
->getChain();
483 SDValue Ptr
= ST
->getBasePtr();
486 unsigned Alignment
= ST
->getAlignment();
487 MachineMemOperand::Flags MMOFlags
= ST
->getMemOperand()->getFlags();
488 AAMDNodes AAInfo
= ST
->getAAInfo();
490 if (!ST
->isTruncatingStore()) {
491 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
492 if (SDNode
*OptStore
= OptimizeFloatStore(ST
).getNode()) {
493 ReplaceNode(ST
, OptStore
);
497 SDValue Value
= ST
->getValue();
498 MVT VT
= Value
.getSimpleValueType();
499 switch (TLI
.getOperationAction(ISD::STORE
, VT
)) {
500 default: llvm_unreachable("This action is not supported yet!");
501 case TargetLowering::Legal
: {
502 // If this is an unaligned store and the target doesn't support it,
504 EVT MemVT
= ST
->getMemoryVT();
505 const DataLayout
&DL
= DAG
.getDataLayout();
506 if (!TLI
.allowsMemoryAccess(*DAG
.getContext(), DL
, MemVT
,
507 *ST
->getMemOperand())) {
508 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
509 SDValue Result
= TLI
.expandUnalignedStore(ST
, DAG
);
510 ReplaceNode(SDValue(ST
, 0), Result
);
512 LLVM_DEBUG(dbgs() << "Legal store\n");
515 case TargetLowering::Custom
: {
516 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
517 SDValue Res
= TLI
.LowerOperation(SDValue(Node
, 0), DAG
);
518 if (Res
&& Res
!= SDValue(Node
, 0))
519 ReplaceNode(SDValue(Node
, 0), Res
);
522 case TargetLowering::Promote
: {
523 MVT NVT
= TLI
.getTypeToPromoteTo(ISD::STORE
, VT
);
524 assert(NVT
.getSizeInBits() == VT
.getSizeInBits() &&
525 "Can only promote stores to same size type");
526 Value
= DAG
.getNode(ISD::BITCAST
, dl
, NVT
, Value
);
528 DAG
.getStore(Chain
, dl
, Value
, Ptr
, ST
->getPointerInfo(),
529 Alignment
, MMOFlags
, AAInfo
);
530 ReplaceNode(SDValue(Node
, 0), Result
);
537 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
538 SDValue Value
= ST
->getValue();
539 EVT StVT
= ST
->getMemoryVT();
540 unsigned StWidth
= StVT
.getSizeInBits();
541 auto &DL
= DAG
.getDataLayout();
543 if (StWidth
!= StVT
.getStoreSizeInBits()) {
544 // Promote to a byte-sized store with upper bits zero if not
545 // storing an integral number of bytes. For example, promote
546 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
547 EVT NVT
= EVT::getIntegerVT(*DAG
.getContext(),
548 StVT
.getStoreSizeInBits());
549 Value
= DAG
.getZeroExtendInReg(Value
, dl
, StVT
);
551 DAG
.getTruncStore(Chain
, dl
, Value
, Ptr
, ST
->getPointerInfo(), NVT
,
552 Alignment
, MMOFlags
, AAInfo
);
553 ReplaceNode(SDValue(Node
, 0), Result
);
554 } else if (StWidth
& (StWidth
- 1)) {
555 // If not storing a power-of-2 number of bits, expand as two stores.
556 assert(!StVT
.isVector() && "Unsupported truncstore!");
557 unsigned LogStWidth
= Log2_32(StWidth
);
558 assert(LogStWidth
< 32);
559 unsigned RoundWidth
= 1 << LogStWidth
;
560 assert(RoundWidth
< StWidth
);
561 unsigned ExtraWidth
= StWidth
- RoundWidth
;
562 assert(ExtraWidth
< RoundWidth
);
563 assert(!(RoundWidth
% 8) && !(ExtraWidth
% 8) &&
564 "Store size not an integral number of bytes!");
565 EVT RoundVT
= EVT::getIntegerVT(*DAG
.getContext(), RoundWidth
);
566 EVT ExtraVT
= EVT::getIntegerVT(*DAG
.getContext(), ExtraWidth
);
568 unsigned IncrementSize
;
570 if (DL
.isLittleEndian()) {
571 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
572 // Store the bottom RoundWidth bits.
573 Lo
= DAG
.getTruncStore(Chain
, dl
, Value
, Ptr
, ST
->getPointerInfo(),
574 RoundVT
, Alignment
, MMOFlags
, AAInfo
);
576 // Store the remaining ExtraWidth bits.
577 IncrementSize
= RoundWidth
/ 8;
578 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
579 DAG
.getConstant(IncrementSize
, dl
,
580 Ptr
.getValueType()));
582 ISD::SRL
, dl
, Value
.getValueType(), Value
,
583 DAG
.getConstant(RoundWidth
, dl
,
584 TLI
.getShiftAmountTy(Value
.getValueType(), DL
)));
585 Hi
= DAG
.getTruncStore(
587 ST
->getPointerInfo().getWithOffset(IncrementSize
), ExtraVT
,
588 MinAlign(Alignment
, IncrementSize
), MMOFlags
, AAInfo
);
590 // Big endian - avoid unaligned stores.
591 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
592 // Store the top RoundWidth bits.
594 ISD::SRL
, dl
, Value
.getValueType(), Value
,
595 DAG
.getConstant(ExtraWidth
, dl
,
596 TLI
.getShiftAmountTy(Value
.getValueType(), DL
)));
597 Hi
= DAG
.getTruncStore(Chain
, dl
, Hi
, Ptr
, ST
->getPointerInfo(),
598 RoundVT
, Alignment
, MMOFlags
, AAInfo
);
600 // Store the remaining ExtraWidth bits.
601 IncrementSize
= RoundWidth
/ 8;
602 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
603 DAG
.getConstant(IncrementSize
, dl
,
604 Ptr
.getValueType()));
605 Lo
= DAG
.getTruncStore(
606 Chain
, dl
, Value
, Ptr
,
607 ST
->getPointerInfo().getWithOffset(IncrementSize
), ExtraVT
,
608 MinAlign(Alignment
, IncrementSize
), MMOFlags
, AAInfo
);
611 // The order of the stores doesn't matter.
612 SDValue Result
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
, Hi
);
613 ReplaceNode(SDValue(Node
, 0), Result
);
615 switch (TLI
.getTruncStoreAction(ST
->getValue().getValueType(), StVT
)) {
616 default: llvm_unreachable("This action is not supported yet!");
617 case TargetLowering::Legal
: {
618 EVT MemVT
= ST
->getMemoryVT();
619 // If this is an unaligned store and the target doesn't support it,
621 if (!TLI
.allowsMemoryAccess(*DAG
.getContext(), DL
, MemVT
,
622 *ST
->getMemOperand())) {
623 SDValue Result
= TLI
.expandUnalignedStore(ST
, DAG
);
624 ReplaceNode(SDValue(ST
, 0), Result
);
628 case TargetLowering::Custom
: {
629 SDValue Res
= TLI
.LowerOperation(SDValue(Node
, 0), DAG
);
630 if (Res
&& Res
!= SDValue(Node
, 0))
631 ReplaceNode(SDValue(Node
, 0), Res
);
634 case TargetLowering::Expand
:
635 assert(!StVT
.isVector() &&
636 "Vector Stores are handled in LegalizeVectorOps");
640 // TRUNCSTORE:i16 i32 -> STORE i16
641 if (TLI
.isTypeLegal(StVT
)) {
642 Value
= DAG
.getNode(ISD::TRUNCATE
, dl
, StVT
, Value
);
643 Result
= DAG
.getStore(Chain
, dl
, Value
, Ptr
, ST
->getPointerInfo(),
644 Alignment
, MMOFlags
, AAInfo
);
646 // The in-memory type isn't legal. Truncate to the type it would promote
647 // to, and then do a truncstore.
648 Value
= DAG
.getNode(ISD::TRUNCATE
, dl
,
649 TLI
.getTypeToTransformTo(*DAG
.getContext(), StVT
),
651 Result
= DAG
.getTruncStore(Chain
, dl
, Value
, Ptr
, ST
->getPointerInfo(),
652 StVT
, Alignment
, MMOFlags
, AAInfo
);
655 ReplaceNode(SDValue(Node
, 0), Result
);
661 void SelectionDAGLegalize::LegalizeLoadOps(SDNode
*Node
) {
662 LoadSDNode
*LD
= cast
<LoadSDNode
>(Node
);
663 SDValue Chain
= LD
->getChain(); // The chain.
664 SDValue Ptr
= LD
->getBasePtr(); // The base pointer.
665 SDValue Value
; // The value returned by the load op.
668 ISD::LoadExtType ExtType
= LD
->getExtensionType();
669 if (ExtType
== ISD::NON_EXTLOAD
) {
670 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
671 MVT VT
= Node
->getSimpleValueType(0);
672 SDValue RVal
= SDValue(Node
, 0);
673 SDValue RChain
= SDValue(Node
, 1);
675 switch (TLI
.getOperationAction(Node
->getOpcode(), VT
)) {
676 default: llvm_unreachable("This action is not supported yet!");
677 case TargetLowering::Legal
: {
678 EVT MemVT
= LD
->getMemoryVT();
679 const DataLayout
&DL
= DAG
.getDataLayout();
680 // If this is an unaligned load and the target doesn't support it,
682 if (!TLI
.allowsMemoryAccess(*DAG
.getContext(), DL
, MemVT
,
683 *LD
->getMemOperand())) {
684 std::tie(RVal
, RChain
) = TLI
.expandUnalignedLoad(LD
, DAG
);
688 case TargetLowering::Custom
:
689 if (SDValue Res
= TLI
.LowerOperation(RVal
, DAG
)) {
691 RChain
= Res
.getValue(1);
695 case TargetLowering::Promote
: {
696 MVT NVT
= TLI
.getTypeToPromoteTo(Node
->getOpcode(), VT
);
697 assert(NVT
.getSizeInBits() == VT
.getSizeInBits() &&
698 "Can only promote loads to same size type");
700 SDValue Res
= DAG
.getLoad(NVT
, dl
, Chain
, Ptr
, LD
->getMemOperand());
701 RVal
= DAG
.getNode(ISD::BITCAST
, dl
, VT
, Res
);
702 RChain
= Res
.getValue(1);
706 if (RChain
.getNode() != Node
) {
707 assert(RVal
.getNode() != Node
&& "Load must be completely replaced");
708 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 0), RVal
);
709 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 1), RChain
);
711 UpdatedNodes
->insert(RVal
.getNode());
712 UpdatedNodes
->insert(RChain
.getNode());
719 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
720 EVT SrcVT
= LD
->getMemoryVT();
721 unsigned SrcWidth
= SrcVT
.getSizeInBits();
722 unsigned Alignment
= LD
->getAlignment();
723 MachineMemOperand::Flags MMOFlags
= LD
->getMemOperand()->getFlags();
724 AAMDNodes AAInfo
= LD
->getAAInfo();
726 if (SrcWidth
!= SrcVT
.getStoreSizeInBits() &&
727 // Some targets pretend to have an i1 loading operation, and actually
728 // load an i8. This trick is correct for ZEXTLOAD because the top 7
729 // bits are guaranteed to be zero; it helps the optimizers understand
730 // that these bits are zero. It is also useful for EXTLOAD, since it
731 // tells the optimizers that those bits are undefined. It would be
732 // nice to have an effective generic way of getting these benefits...
733 // Until such a way is found, don't insist on promoting i1 here.
735 TLI
.getLoadExtAction(ExtType
, Node
->getValueType(0), MVT::i1
) ==
736 TargetLowering::Promote
)) {
737 // Promote to a byte-sized load if not loading an integral number of
738 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
739 unsigned NewWidth
= SrcVT
.getStoreSizeInBits();
740 EVT NVT
= EVT::getIntegerVT(*DAG
.getContext(), NewWidth
);
743 // The extra bits are guaranteed to be zero, since we stored them that
744 // way. A zext load from NVT thus automatically gives zext from SrcVT.
746 ISD::LoadExtType NewExtType
=
747 ExtType
== ISD::ZEXTLOAD
? ISD::ZEXTLOAD
: ISD::EXTLOAD
;
750 DAG
.getExtLoad(NewExtType
, dl
, Node
->getValueType(0), Chain
, Ptr
,
751 LD
->getPointerInfo(), NVT
, Alignment
, MMOFlags
, AAInfo
);
753 Ch
= Result
.getValue(1); // The chain.
755 if (ExtType
== ISD::SEXTLOAD
)
756 // Having the top bits zero doesn't help when sign extending.
757 Result
= DAG
.getNode(ISD::SIGN_EXTEND_INREG
, dl
,
758 Result
.getValueType(),
759 Result
, DAG
.getValueType(SrcVT
));
760 else if (ExtType
== ISD::ZEXTLOAD
|| NVT
== Result
.getValueType())
761 // All the top bits are guaranteed to be zero - inform the optimizers.
762 Result
= DAG
.getNode(ISD::AssertZext
, dl
,
763 Result
.getValueType(), Result
,
764 DAG
.getValueType(SrcVT
));
768 } else if (SrcWidth
& (SrcWidth
- 1)) {
769 // If not loading a power-of-2 number of bits, expand as two loads.
770 assert(!SrcVT
.isVector() && "Unsupported extload!");
771 unsigned LogSrcWidth
= Log2_32(SrcWidth
);
772 assert(LogSrcWidth
< 32);
773 unsigned RoundWidth
= 1 << LogSrcWidth
;
774 assert(RoundWidth
< SrcWidth
);
775 unsigned ExtraWidth
= SrcWidth
- RoundWidth
;
776 assert(ExtraWidth
< RoundWidth
);
777 assert(!(RoundWidth
% 8) && !(ExtraWidth
% 8) &&
778 "Load size not an integral number of bytes!");
779 EVT RoundVT
= EVT::getIntegerVT(*DAG
.getContext(), RoundWidth
);
780 EVT ExtraVT
= EVT::getIntegerVT(*DAG
.getContext(), ExtraWidth
);
782 unsigned IncrementSize
;
783 auto &DL
= DAG
.getDataLayout();
785 if (DL
.isLittleEndian()) {
786 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
787 // Load the bottom RoundWidth bits.
788 Lo
= DAG
.getExtLoad(ISD::ZEXTLOAD
, dl
, Node
->getValueType(0), Chain
, Ptr
,
789 LD
->getPointerInfo(), RoundVT
, Alignment
, MMOFlags
,
792 // Load the remaining ExtraWidth bits.
793 IncrementSize
= RoundWidth
/ 8;
794 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
795 DAG
.getConstant(IncrementSize
, dl
,
796 Ptr
.getValueType()));
797 Hi
= DAG
.getExtLoad(ExtType
, dl
, Node
->getValueType(0), Chain
, Ptr
,
798 LD
->getPointerInfo().getWithOffset(IncrementSize
),
799 ExtraVT
, MinAlign(Alignment
, IncrementSize
), MMOFlags
,
802 // Build a factor node to remember that this load is independent of
804 Ch
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
.getValue(1),
807 // Move the top bits to the right place.
809 ISD::SHL
, dl
, Hi
.getValueType(), Hi
,
810 DAG
.getConstant(RoundWidth
, dl
,
811 TLI
.getShiftAmountTy(Hi
.getValueType(), DL
)));
813 // Join the hi and lo parts.
814 Value
= DAG
.getNode(ISD::OR
, dl
, Node
->getValueType(0), Lo
, Hi
);
816 // Big endian - avoid unaligned loads.
817 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
818 // Load the top RoundWidth bits.
819 Hi
= DAG
.getExtLoad(ExtType
, dl
, Node
->getValueType(0), Chain
, Ptr
,
820 LD
->getPointerInfo(), RoundVT
, Alignment
, MMOFlags
,
823 // Load the remaining ExtraWidth bits.
824 IncrementSize
= RoundWidth
/ 8;
825 Ptr
= DAG
.getNode(ISD::ADD
, dl
, Ptr
.getValueType(), Ptr
,
826 DAG
.getConstant(IncrementSize
, dl
,
827 Ptr
.getValueType()));
828 Lo
= DAG
.getExtLoad(ISD::ZEXTLOAD
, dl
, Node
->getValueType(0), Chain
, Ptr
,
829 LD
->getPointerInfo().getWithOffset(IncrementSize
),
830 ExtraVT
, MinAlign(Alignment
, IncrementSize
), MMOFlags
,
833 // Build a factor node to remember that this load is independent of
835 Ch
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Lo
.getValue(1),
838 // Move the top bits to the right place.
840 ISD::SHL
, dl
, Hi
.getValueType(), Hi
,
841 DAG
.getConstant(ExtraWidth
, dl
,
842 TLI
.getShiftAmountTy(Hi
.getValueType(), DL
)));
844 // Join the hi and lo parts.
845 Value
= DAG
.getNode(ISD::OR
, dl
, Node
->getValueType(0), Lo
, Hi
);
850 bool isCustom
= false;
851 switch (TLI
.getLoadExtAction(ExtType
, Node
->getValueType(0),
852 SrcVT
.getSimpleVT())) {
853 default: llvm_unreachable("This action is not supported yet!");
854 case TargetLowering::Custom
:
857 case TargetLowering::Legal
:
858 Value
= SDValue(Node
, 0);
859 Chain
= SDValue(Node
, 1);
862 if (SDValue Res
= TLI
.LowerOperation(SDValue(Node
, 0), DAG
)) {
864 Chain
= Res
.getValue(1);
867 // If this is an unaligned load and the target doesn't support it,
869 EVT MemVT
= LD
->getMemoryVT();
870 const DataLayout
&DL
= DAG
.getDataLayout();
871 if (!TLI
.allowsMemoryAccess(*DAG
.getContext(), DL
, MemVT
,
872 *LD
->getMemOperand())) {
873 std::tie(Value
, Chain
) = TLI
.expandUnalignedLoad(LD
, DAG
);
878 case TargetLowering::Expand
: {
879 EVT DestVT
= Node
->getValueType(0);
880 if (!TLI
.isLoadExtLegal(ISD::EXTLOAD
, DestVT
, SrcVT
)) {
881 // If the source type is not legal, see if there is a legal extload to
882 // an intermediate type that we can then extend further.
883 EVT LoadVT
= TLI
.getRegisterType(SrcVT
.getSimpleVT());
884 if (TLI
.isTypeLegal(SrcVT
) || // Same as SrcVT == LoadVT?
885 TLI
.isLoadExtLegal(ExtType
, LoadVT
, SrcVT
)) {
886 // If we are loading a legal type, this is a non-extload followed by a
888 ISD::LoadExtType MidExtType
=
889 (LoadVT
== SrcVT
) ? ISD::NON_EXTLOAD
: ExtType
;
891 SDValue Load
= DAG
.getExtLoad(MidExtType
, dl
, LoadVT
, Chain
, Ptr
,
892 SrcVT
, LD
->getMemOperand());
894 ISD::getExtForLoadExtType(SrcVT
.isFloatingPoint(), ExtType
);
895 Value
= DAG
.getNode(ExtendOp
, dl
, Node
->getValueType(0), Load
);
896 Chain
= Load
.getValue(1);
900 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
901 // normal undefined upper bits behavior to allow using an in-reg extend
902 // with the illegal FP type, so load as an integer and do the
903 // from-integer conversion.
904 if (SrcVT
.getScalarType() == MVT::f16
) {
905 EVT ISrcVT
= SrcVT
.changeTypeToInteger();
906 EVT IDestVT
= DestVT
.changeTypeToInteger();
907 EVT ILoadVT
= TLI
.getRegisterType(IDestVT
.getSimpleVT());
909 SDValue Result
= DAG
.getExtLoad(ISD::ZEXTLOAD
, dl
, ILoadVT
, Chain
,
910 Ptr
, ISrcVT
, LD
->getMemOperand());
911 Value
= DAG
.getNode(ISD::FP16_TO_FP
, dl
, DestVT
, Result
);
912 Chain
= Result
.getValue(1);
917 assert(!SrcVT
.isVector() &&
918 "Vector Loads are handled in LegalizeVectorOps");
920 // FIXME: This does not work for vectors on most targets. Sign-
921 // and zero-extend operations are currently folded into extending
922 // loads, whether they are legal or not, and then we end up here
923 // without any support for legalizing them.
924 assert(ExtType
!= ISD::EXTLOAD
&&
925 "EXTLOAD should always be supported!");
926 // Turn the unsupported load into an EXTLOAD followed by an
927 // explicit zero/sign extend inreg.
928 SDValue Result
= DAG
.getExtLoad(ISD::EXTLOAD
, dl
,
929 Node
->getValueType(0),
931 LD
->getMemOperand());
933 if (ExtType
== ISD::SEXTLOAD
)
934 ValRes
= DAG
.getNode(ISD::SIGN_EXTEND_INREG
, dl
,
935 Result
.getValueType(),
936 Result
, DAG
.getValueType(SrcVT
));
938 ValRes
= DAG
.getZeroExtendInReg(Result
, dl
, SrcVT
.getScalarType());
940 Chain
= Result
.getValue(1);
946 // Since loads produce two values, make sure to remember that we legalized
948 if (Chain
.getNode() != Node
) {
949 assert(Value
.getNode() != Node
&& "Load must be completely replaced");
950 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 0), Value
);
951 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 1), Chain
);
953 UpdatedNodes
->insert(Value
.getNode());
954 UpdatedNodes
->insert(Chain
.getNode());
960 /// Return a legal replacement for the given operation, with all legal operands.
961 void SelectionDAGLegalize::LegalizeOp(SDNode
*Node
) {
962 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node
->dump(&DAG
));
964 // Allow illegal target nodes and illegal registers.
965 if (Node
->getOpcode() == ISD::TargetConstant
||
966 Node
->getOpcode() == ISD::Register
)
970 for (unsigned i
= 0, e
= Node
->getNumValues(); i
!= e
; ++i
)
971 assert((TLI
.getTypeAction(*DAG
.getContext(), Node
->getValueType(i
)) ==
972 TargetLowering::TypeLegal
||
973 TLI
.isTypeLegal(Node
->getValueType(i
))) &&
974 "Unexpected illegal type!");
976 for (const SDValue
&Op
: Node
->op_values())
977 assert((TLI
.getTypeAction(*DAG
.getContext(), Op
.getValueType()) ==
978 TargetLowering::TypeLegal
||
979 TLI
.isTypeLegal(Op
.getValueType()) ||
980 Op
.getOpcode() == ISD::TargetConstant
||
981 Op
.getOpcode() == ISD::Register
) &&
982 "Unexpected illegal type!");
985 // Figure out the correct action; the way to query this varies by opcode
986 TargetLowering::LegalizeAction Action
= TargetLowering::Legal
;
987 bool SimpleFinishLegalizing
= true;
988 switch (Node
->getOpcode()) {
989 case ISD::INTRINSIC_W_CHAIN
:
990 case ISD::INTRINSIC_WO_CHAIN
:
991 case ISD::INTRINSIC_VOID
:
993 Action
= TLI
.getOperationAction(Node
->getOpcode(), MVT::Other
);
995 case ISD::GET_DYNAMIC_AREA_OFFSET
:
996 Action
= TLI
.getOperationAction(Node
->getOpcode(),
997 Node
->getValueType(0));
1000 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1001 Node
->getValueType(0));
1002 if (Action
!= TargetLowering::Promote
)
1003 Action
= TLI
.getOperationAction(Node
->getOpcode(), MVT::Other
);
1005 case ISD::FP_TO_FP16
:
1006 case ISD::SINT_TO_FP
:
1007 case ISD::UINT_TO_FP
:
1008 case ISD::EXTRACT_VECTOR_ELT
:
1013 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1014 Node
->getOperand(0).getValueType());
1016 case ISD::FP_ROUND_INREG
:
1017 case ISD::SIGN_EXTEND_INREG
: {
1018 EVT InnerType
= cast
<VTSDNode
>(Node
->getOperand(1))->getVT();
1019 Action
= TLI
.getOperationAction(Node
->getOpcode(), InnerType
);
1022 case ISD::ATOMIC_STORE
:
1023 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1024 Node
->getOperand(2).getValueType());
1026 case ISD::SELECT_CC
:
1029 unsigned CCOperand
= Node
->getOpcode() == ISD::SELECT_CC
? 4 :
1030 Node
->getOpcode() == ISD::SETCC
? 2 : 1;
1031 unsigned CompareOperand
= Node
->getOpcode() == ISD::BR_CC
? 2 : 0;
1032 MVT OpVT
= Node
->getOperand(CompareOperand
).getSimpleValueType();
1033 ISD::CondCode CCCode
=
1034 cast
<CondCodeSDNode
>(Node
->getOperand(CCOperand
))->get();
1035 Action
= TLI
.getCondCodeAction(CCCode
, OpVT
);
1036 if (Action
== TargetLowering::Legal
) {
1037 if (Node
->getOpcode() == ISD::SELECT_CC
)
1038 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1039 Node
->getValueType(0));
1041 Action
= TLI
.getOperationAction(Node
->getOpcode(), OpVT
);
1047 // FIXME: Model these properly. LOAD and STORE are complicated, and
1048 // STORE expects the unlegalized operand in some cases.
1049 SimpleFinishLegalizing
= false;
1051 case ISD::CALLSEQ_START
:
1052 case ISD::CALLSEQ_END
:
1053 // FIXME: This shouldn't be necessary. These nodes have special properties
1054 // dealing with the recursive nature of legalization. Removing this
1055 // special case should be done as part of making LegalizeDAG non-recursive.
1056 SimpleFinishLegalizing
= false;
1058 case ISD::EXTRACT_ELEMENT
:
1059 case ISD::FLT_ROUNDS_
:
1060 case ISD::MERGE_VALUES
:
1061 case ISD::EH_RETURN
:
1062 case ISD::FRAME_TO_ARGS_OFFSET
:
1063 case ISD::EH_DWARF_CFA
:
1064 case ISD::EH_SJLJ_SETJMP
:
1065 case ISD::EH_SJLJ_LONGJMP
:
1066 case ISD::EH_SJLJ_SETUP_DISPATCH
:
1067 // These operations lie about being legal: when they claim to be legal,
1068 // they should actually be expanded.
1069 Action
= TLI
.getOperationAction(Node
->getOpcode(), Node
->getValueType(0));
1070 if (Action
== TargetLowering::Legal
)
1071 Action
= TargetLowering::Expand
;
1073 case ISD::INIT_TRAMPOLINE
:
1074 case ISD::ADJUST_TRAMPOLINE
:
1075 case ISD::FRAMEADDR
:
1076 case ISD::RETURNADDR
:
1077 case ISD::ADDROFRETURNADDR
:
1078 case ISD::SPONENTRY
:
1079 // These operations lie about being legal: when they claim to be legal,
1080 // they should actually be custom-lowered.
1081 Action
= TLI
.getOperationAction(Node
->getOpcode(), Node
->getValueType(0));
1082 if (Action
== TargetLowering::Legal
)
1083 Action
= TargetLowering::Custom
;
1085 case ISD::READCYCLECOUNTER
:
1086 // READCYCLECOUNTER returns an i64, even if type legalization might have
1087 // expanded that to several smaller types.
1088 Action
= TLI
.getOperationAction(Node
->getOpcode(), MVT::i64
);
1090 case ISD::READ_REGISTER
:
1091 case ISD::WRITE_REGISTER
:
1092 // Named register is legal in the DAG, but blocked by register name
1093 // selection if not implemented by target (to chose the correct register)
1094 // They'll be converted to Copy(To/From)Reg.
1095 Action
= TargetLowering::Legal
;
1097 case ISD::DEBUGTRAP
:
1098 Action
= TLI
.getOperationAction(Node
->getOpcode(), Node
->getValueType(0));
1099 if (Action
== TargetLowering::Expand
) {
1100 // replace ISD::DEBUGTRAP with ISD::TRAP
1102 NewVal
= DAG
.getNode(ISD::TRAP
, SDLoc(Node
), Node
->getVTList(),
1103 Node
->getOperand(0));
1104 ReplaceNode(Node
, NewVal
.getNode());
1105 LegalizeOp(NewVal
.getNode());
1112 case ISD::USUBSAT
: {
1113 Action
= TLI
.getOperationAction(Node
->getOpcode(), Node
->getValueType(0));
1117 case ISD::SMULFIXSAT
:
1118 case ISD::UMULFIX
: {
1119 unsigned Scale
= Node
->getConstantOperandVal(2);
1120 Action
= TLI
.getFixedPointOperationAction(Node
->getOpcode(),
1121 Node
->getValueType(0), Scale
);
1125 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1126 cast
<MaskedScatterSDNode
>(Node
)->getValue().getValueType());
1129 Action
= TLI
.getOperationAction(Node
->getOpcode(),
1130 cast
<MaskedStoreSDNode
>(Node
)->getValue().getValueType());
1132 case ISD::VECREDUCE_FADD
:
1133 case ISD::VECREDUCE_FMUL
:
1134 case ISD::VECREDUCE_ADD
:
1135 case ISD::VECREDUCE_MUL
:
1136 case ISD::VECREDUCE_AND
:
1137 case ISD::VECREDUCE_OR
:
1138 case ISD::VECREDUCE_XOR
:
1139 case ISD::VECREDUCE_SMAX
:
1140 case ISD::VECREDUCE_SMIN
:
1141 case ISD::VECREDUCE_UMAX
:
1142 case ISD::VECREDUCE_UMIN
:
1143 case ISD::VECREDUCE_FMAX
:
1144 case ISD::VECREDUCE_FMIN
:
1145 Action
= TLI
.getOperationAction(
1146 Node
->getOpcode(), Node
->getOperand(0).getValueType());
1149 if (Node
->getOpcode() >= ISD::BUILTIN_OP_END
) {
1150 Action
= TargetLowering::Legal
;
1152 Action
= TLI
.getOperationAction(Node
->getOpcode(), Node
->getValueType(0));
1157 if (SimpleFinishLegalizing
) {
1158 SDNode
*NewNode
= Node
;
1159 switch (Node
->getOpcode()) {
1166 // Legalizing shifts/rotates requires adjusting the shift amount
1167 // to the appropriate width.
1168 SDValue Op0
= Node
->getOperand(0);
1169 SDValue Op1
= Node
->getOperand(1);
1170 if (!Op1
.getValueType().isVector()) {
1171 SDValue SAO
= DAG
.getShiftAmountOperand(Op0
.getValueType(), Op1
);
1172 // The getShiftAmountOperand() may create a new operand node or
1173 // return the existing one. If new operand is created we need
1174 // to update the parent node.
1175 // Do not try to legalize SAO here! It will be automatically legalized
1176 // in the next round.
1178 NewNode
= DAG
.UpdateNodeOperands(Node
, Op0
, SAO
);
1184 case ISD::SRL_PARTS
:
1185 case ISD::SRA_PARTS
:
1186 case ISD::SHL_PARTS
: {
1187 // Legalizing shifts/rotates requires adjusting the shift amount
1188 // to the appropriate width.
1189 SDValue Op0
= Node
->getOperand(0);
1190 SDValue Op1
= Node
->getOperand(1);
1191 SDValue Op2
= Node
->getOperand(2);
1192 if (!Op2
.getValueType().isVector()) {
1193 SDValue SAO
= DAG
.getShiftAmountOperand(Op0
.getValueType(), Op2
);
1194 // The getShiftAmountOperand() may create a new operand node or
1195 // return the existing one. If new operand is created we need
1196 // to update the parent node.
1198 NewNode
= DAG
.UpdateNodeOperands(Node
, Op0
, Op1
, SAO
);
1204 if (NewNode
!= Node
) {
1205 ReplaceNode(Node
, NewNode
);
1209 case TargetLowering::Legal
:
1210 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1212 case TargetLowering::Custom
:
1213 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1214 // FIXME: The handling for custom lowering with multiple results is
1216 if (SDValue Res
= TLI
.LowerOperation(SDValue(Node
, 0), DAG
)) {
1217 if (!(Res
.getNode() != Node
|| Res
.getResNo() != 0))
1220 if (Node
->getNumValues() == 1) {
1221 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1222 // We can just directly replace this node with the lowered value.
1223 ReplaceNode(SDValue(Node
, 0), Res
);
1227 SmallVector
<SDValue
, 8> ResultVals
;
1228 for (unsigned i
= 0, e
= Node
->getNumValues(); i
!= e
; ++i
)
1229 ResultVals
.push_back(Res
.getValue(i
));
1230 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1231 ReplaceNode(Node
, ResultVals
.data());
1234 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1236 case TargetLowering::Expand
:
1237 if (ExpandNode(Node
))
1240 case TargetLowering::LibCall
:
1241 ConvertNodeToLibcall(Node
);
1243 case TargetLowering::Promote
:
1249 switch (Node
->getOpcode()) {
1256 llvm_unreachable("Do not know how to legalize this operator!");
1258 case ISD::CALLSEQ_START
:
1259 case ISD::CALLSEQ_END
:
1262 return LegalizeLoadOps(Node
);
1264 return LegalizeStoreOps(Node
);
1268 SDValue
SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op
) {
1269 SDValue Vec
= Op
.getOperand(0);
1270 SDValue Idx
= Op
.getOperand(1);
1273 // Before we generate a new store to a temporary stack slot, see if there is
1274 // already one that we can use. There often is because when we scalarize
1275 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1276 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1277 // the vector. If all are expanded here, we don't want one store per vector
1280 // Caches for hasPredecessorHelper
1281 SmallPtrSet
<const SDNode
*, 32> Visited
;
1282 SmallVector
<const SDNode
*, 16> Worklist
;
1283 Visited
.insert(Op
.getNode());
1284 Worklist
.push_back(Idx
.getNode());
1285 SDValue StackPtr
, Ch
;
1286 for (SDNode::use_iterator UI
= Vec
.getNode()->use_begin(),
1287 UE
= Vec
.getNode()->use_end(); UI
!= UE
; ++UI
) {
1289 if (StoreSDNode
*ST
= dyn_cast
<StoreSDNode
>(User
)) {
1290 if (ST
->isIndexed() || ST
->isTruncatingStore() ||
1291 ST
->getValue() != Vec
)
1294 // Make sure that nothing else could have stored into the destination of
1296 if (!ST
->getChain().reachesChainWithoutSideEffects(DAG
.getEntryNode()))
1299 // If the index is dependent on the store we will introduce a cycle when
1300 // creating the load (the load uses the index, and by replacing the chain
1301 // we will make the index dependent on the load). Also, the store might be
1302 // dependent on the extractelement and introduce a cycle when creating
1304 if (SDNode::hasPredecessorHelper(ST
, Visited
, Worklist
) ||
1305 ST
->hasPredecessor(Op
.getNode()))
1308 StackPtr
= ST
->getBasePtr();
1309 Ch
= SDValue(ST
, 0);
1314 EVT VecVT
= Vec
.getValueType();
1316 if (!Ch
.getNode()) {
1317 // Store the value to a temporary stack slot, then LOAD the returned part.
1318 StackPtr
= DAG
.CreateStackTemporary(VecVT
);
1319 Ch
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
,
1320 MachinePointerInfo());
1323 StackPtr
= TLI
.getVectorElementPointer(DAG
, StackPtr
, VecVT
, Idx
);
1327 if (Op
.getValueType().isVector())
1329 DAG
.getLoad(Op
.getValueType(), dl
, Ch
, StackPtr
, MachinePointerInfo());
1331 NewLoad
= DAG
.getExtLoad(ISD::EXTLOAD
, dl
, Op
.getValueType(), Ch
, StackPtr
,
1332 MachinePointerInfo(),
1333 VecVT
.getVectorElementType());
1335 // Replace the chain going out of the store, by the one out of the load.
1336 DAG
.ReplaceAllUsesOfValueWith(Ch
, SDValue(NewLoad
.getNode(), 1));
1338 // We introduced a cycle though, so update the loads operands, making sure
1339 // to use the original store's chain as an incoming chain.
1340 SmallVector
<SDValue
, 6> NewLoadOperands(NewLoad
->op_begin(),
1342 NewLoadOperands
[0] = Ch
;
1344 SDValue(DAG
.UpdateNodeOperands(NewLoad
.getNode(), NewLoadOperands
), 0);
1348 SDValue
SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op
) {
1349 assert(Op
.getValueType().isVector() && "Non-vector insert subvector!");
1351 SDValue Vec
= Op
.getOperand(0);
1352 SDValue Part
= Op
.getOperand(1);
1353 SDValue Idx
= Op
.getOperand(2);
1356 // Store the value to a temporary stack slot, then LOAD the returned part.
1357 EVT VecVT
= Vec
.getValueType();
1358 SDValue StackPtr
= DAG
.CreateStackTemporary(VecVT
);
1359 int FI
= cast
<FrameIndexSDNode
>(StackPtr
.getNode())->getIndex();
1360 MachinePointerInfo PtrInfo
=
1361 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), FI
);
1363 // First store the whole vector.
1364 SDValue Ch
= DAG
.getStore(DAG
.getEntryNode(), dl
, Vec
, StackPtr
, PtrInfo
);
1366 // Then store the inserted part.
1367 SDValue SubStackPtr
= TLI
.getVectorElementPointer(DAG
, StackPtr
, VecVT
, Idx
);
1369 // Store the subvector.
1370 Ch
= DAG
.getStore(Ch
, dl
, Part
, SubStackPtr
, MachinePointerInfo());
1372 // Finally, load the updated vector.
1373 return DAG
.getLoad(Op
.getValueType(), dl
, Ch
, StackPtr
, PtrInfo
);
1376 SDValue
SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode
* Node
) {
1377 // We can't handle this case efficiently. Allocate a sufficiently
1378 // aligned object on the stack, store each element into it, then load
1379 // the result as a vector.
1380 // Create the stack frame object.
1381 EVT VT
= Node
->getValueType(0);
1382 EVT EltVT
= VT
.getVectorElementType();
1384 SDValue FIPtr
= DAG
.CreateStackTemporary(VT
);
1385 int FI
= cast
<FrameIndexSDNode
>(FIPtr
.getNode())->getIndex();
1386 MachinePointerInfo PtrInfo
=
1387 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), FI
);
1389 // Emit a store of each element to the stack slot.
1390 SmallVector
<SDValue
, 8> Stores
;
1391 unsigned TypeByteSize
= EltVT
.getSizeInBits() / 8;
1392 assert(TypeByteSize
> 0 && "Vector element type too small for stack store!");
1393 // Store (in the right endianness) the elements to memory.
1394 for (unsigned i
= 0, e
= Node
->getNumOperands(); i
!= e
; ++i
) {
1395 // Ignore undef elements.
1396 if (Node
->getOperand(i
).isUndef()) continue;
1398 unsigned Offset
= TypeByteSize
*i
;
1400 SDValue Idx
= DAG
.getConstant(Offset
, dl
, FIPtr
.getValueType());
1401 Idx
= DAG
.getNode(ISD::ADD
, dl
, FIPtr
.getValueType(), FIPtr
, Idx
);
1403 // If the destination vector element type is narrower than the source
1404 // element type, only store the bits necessary.
1405 if (EltVT
.bitsLT(Node
->getOperand(i
).getValueType().getScalarType())) {
1406 Stores
.push_back(DAG
.getTruncStore(DAG
.getEntryNode(), dl
,
1407 Node
->getOperand(i
), Idx
,
1408 PtrInfo
.getWithOffset(Offset
), EltVT
));
1410 Stores
.push_back(DAG
.getStore(DAG
.getEntryNode(), dl
, Node
->getOperand(i
),
1411 Idx
, PtrInfo
.getWithOffset(Offset
)));
1415 if (!Stores
.empty()) // Not all undef elements?
1416 StoreChain
= DAG
.getNode(ISD::TokenFactor
, dl
, MVT::Other
, Stores
);
1418 StoreChain
= DAG
.getEntryNode();
1420 // Result is a load from the stack slot.
1421 return DAG
.getLoad(VT
, dl
, StoreChain
, FIPtr
, PtrInfo
);
1424 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1425 /// containing the sign bit if the target has no integer value capable of
1426 /// holding all bits of the floating-point value.
1427 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt
&State
,
1429 SDValue Value
) const {
1430 EVT FloatVT
= Value
.getValueType();
1431 unsigned NumBits
= FloatVT
.getSizeInBits();
1432 State
.FloatVT
= FloatVT
;
1433 EVT IVT
= EVT::getIntegerVT(*DAG
.getContext(), NumBits
);
1434 // Convert to an integer of the same size.
1435 if (TLI
.isTypeLegal(IVT
)) {
1436 State
.IntValue
= DAG
.getNode(ISD::BITCAST
, DL
, IVT
, Value
);
1437 State
.SignMask
= APInt::getSignMask(NumBits
);
1438 State
.SignBit
= NumBits
- 1;
1442 auto &DataLayout
= DAG
.getDataLayout();
1443 // Store the float to memory, then load the sign part out as an integer.
1444 MVT LoadTy
= TLI
.getRegisterType(*DAG
.getContext(), MVT::i8
);
1445 // First create a temporary that is aligned for both the load and store.
1446 SDValue StackPtr
= DAG
.CreateStackTemporary(FloatVT
, LoadTy
);
1447 int FI
= cast
<FrameIndexSDNode
>(StackPtr
.getNode())->getIndex();
1448 // Then store the float to it.
1449 State
.FloatPtr
= StackPtr
;
1450 MachineFunction
&MF
= DAG
.getMachineFunction();
1451 State
.FloatPointerInfo
= MachinePointerInfo::getFixedStack(MF
, FI
);
1452 State
.Chain
= DAG
.getStore(DAG
.getEntryNode(), DL
, Value
, State
.FloatPtr
,
1453 State
.FloatPointerInfo
);
1456 if (DataLayout
.isBigEndian()) {
1457 assert(FloatVT
.isByteSized() && "Unsupported floating point type!");
1458 // Load out a legal integer with the same sign bit as the float.
1460 State
.IntPointerInfo
= State
.FloatPointerInfo
;
1462 // Advance the pointer so that the loaded byte will contain the sign bit.
1463 unsigned ByteOffset
= (FloatVT
.getSizeInBits() / 8) - 1;
1464 IntPtr
= DAG
.getNode(ISD::ADD
, DL
, StackPtr
.getValueType(), StackPtr
,
1465 DAG
.getConstant(ByteOffset
, DL
, StackPtr
.getValueType()));
1466 State
.IntPointerInfo
= MachinePointerInfo::getFixedStack(MF
, FI
,
1470 State
.IntPtr
= IntPtr
;
1471 State
.IntValue
= DAG
.getExtLoad(ISD::EXTLOAD
, DL
, LoadTy
, State
.Chain
, IntPtr
,
1472 State
.IntPointerInfo
, MVT::i8
);
1473 State
.SignMask
= APInt::getOneBitSet(LoadTy
.getSizeInBits(), 7);
1477 /// Replace the integer value produced by getSignAsIntValue() with a new value
1478 /// and cast the result back to a floating-point type.
1479 SDValue
SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt
&State
,
1481 SDValue NewIntValue
) const {
1483 return DAG
.getNode(ISD::BITCAST
, DL
, State
.FloatVT
, NewIntValue
);
1485 // Override the part containing the sign bit in the value stored on the stack.
1486 SDValue Chain
= DAG
.getTruncStore(State
.Chain
, DL
, NewIntValue
, State
.IntPtr
,
1487 State
.IntPointerInfo
, MVT::i8
);
1488 return DAG
.getLoad(State
.FloatVT
, DL
, Chain
, State
.FloatPtr
,
1489 State
.FloatPointerInfo
);
1492 SDValue
SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode
*Node
) const {
1494 SDValue Mag
= Node
->getOperand(0);
1495 SDValue Sign
= Node
->getOperand(1);
1497 // Get sign bit into an integer value.
1498 FloatSignAsInt SignAsInt
;
1499 getSignAsIntValue(SignAsInt
, DL
, Sign
);
1501 EVT IntVT
= SignAsInt
.IntValue
.getValueType();
1502 SDValue SignMask
= DAG
.getConstant(SignAsInt
.SignMask
, DL
, IntVT
);
1503 SDValue SignBit
= DAG
.getNode(ISD::AND
, DL
, IntVT
, SignAsInt
.IntValue
,
1506 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1507 EVT FloatVT
= Mag
.getValueType();
1508 if (TLI
.isOperationLegalOrCustom(ISD::FABS
, FloatVT
) &&
1509 TLI
.isOperationLegalOrCustom(ISD::FNEG
, FloatVT
)) {
1510 SDValue AbsValue
= DAG
.getNode(ISD::FABS
, DL
, FloatVT
, Mag
);
1511 SDValue NegValue
= DAG
.getNode(ISD::FNEG
, DL
, FloatVT
, AbsValue
);
1512 SDValue Cond
= DAG
.getSetCC(DL
, getSetCCResultType(IntVT
), SignBit
,
1513 DAG
.getConstant(0, DL
, IntVT
), ISD::SETNE
);
1514 return DAG
.getSelect(DL
, FloatVT
, Cond
, NegValue
, AbsValue
);
1517 // Transform Mag value to integer, and clear the sign bit.
1518 FloatSignAsInt MagAsInt
;
1519 getSignAsIntValue(MagAsInt
, DL
, Mag
);
1520 EVT MagVT
= MagAsInt
.IntValue
.getValueType();
1521 SDValue ClearSignMask
= DAG
.getConstant(~MagAsInt
.SignMask
, DL
, MagVT
);
1522 SDValue ClearedSign
= DAG
.getNode(ISD::AND
, DL
, MagVT
, MagAsInt
.IntValue
,
1525 // Get the signbit at the right position for MagAsInt.
1526 int ShiftAmount
= SignAsInt
.SignBit
- MagAsInt
.SignBit
;
1527 EVT ShiftVT
= IntVT
;
1528 if (SignBit
.getValueSizeInBits() < ClearedSign
.getValueSizeInBits()) {
1529 SignBit
= DAG
.getNode(ISD::ZERO_EXTEND
, DL
, MagVT
, SignBit
);
1532 if (ShiftAmount
> 0) {
1533 SDValue ShiftCnst
= DAG
.getConstant(ShiftAmount
, DL
, ShiftVT
);
1534 SignBit
= DAG
.getNode(ISD::SRL
, DL
, ShiftVT
, SignBit
, ShiftCnst
);
1535 } else if (ShiftAmount
< 0) {
1536 SDValue ShiftCnst
= DAG
.getConstant(-ShiftAmount
, DL
, ShiftVT
);
1537 SignBit
= DAG
.getNode(ISD::SHL
, DL
, ShiftVT
, SignBit
, ShiftCnst
);
1539 if (SignBit
.getValueSizeInBits() > ClearedSign
.getValueSizeInBits()) {
1540 SignBit
= DAG
.getNode(ISD::TRUNCATE
, DL
, MagVT
, SignBit
);
1543 // Store the part with the modified sign and convert back to float.
1544 SDValue CopiedSign
= DAG
.getNode(ISD::OR
, DL
, MagVT
, ClearedSign
, SignBit
);
1545 return modifySignAsInt(MagAsInt
, DL
, CopiedSign
);
1548 SDValue
SelectionDAGLegalize::ExpandFABS(SDNode
*Node
) const {
1550 SDValue Value
= Node
->getOperand(0);
1552 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1553 EVT FloatVT
= Value
.getValueType();
1554 if (TLI
.isOperationLegalOrCustom(ISD::FCOPYSIGN
, FloatVT
)) {
1555 SDValue Zero
= DAG
.getConstantFP(0.0, DL
, FloatVT
);
1556 return DAG
.getNode(ISD::FCOPYSIGN
, DL
, FloatVT
, Value
, Zero
);
1559 // Transform value to integer, clear the sign bit and transform back.
1560 FloatSignAsInt ValueAsInt
;
1561 getSignAsIntValue(ValueAsInt
, DL
, Value
);
1562 EVT IntVT
= ValueAsInt
.IntValue
.getValueType();
1563 SDValue ClearSignMask
= DAG
.getConstant(~ValueAsInt
.SignMask
, DL
, IntVT
);
1564 SDValue ClearedSign
= DAG
.getNode(ISD::AND
, DL
, IntVT
, ValueAsInt
.IntValue
,
1566 return modifySignAsInt(ValueAsInt
, DL
, ClearedSign
);
1569 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode
* Node
,
1570 SmallVectorImpl
<SDValue
> &Results
) {
1571 unsigned SPReg
= TLI
.getStackPointerRegisterToSaveRestore();
1572 assert(SPReg
&& "Target cannot require DYNAMIC_STACKALLOC expansion and"
1573 " not tell us which reg is the stack pointer!");
1575 EVT VT
= Node
->getValueType(0);
1576 SDValue Tmp1
= SDValue(Node
, 0);
1577 SDValue Tmp2
= SDValue(Node
, 1);
1578 SDValue Tmp3
= Node
->getOperand(2);
1579 SDValue Chain
= Tmp1
.getOperand(0);
1581 // Chain the dynamic stack allocation so that it doesn't modify the stack
1582 // pointer when other instructions are using the stack.
1583 Chain
= DAG
.getCALLSEQ_START(Chain
, 0, 0, dl
);
1585 SDValue Size
= Tmp2
.getOperand(1);
1586 SDValue SP
= DAG
.getCopyFromReg(Chain
, dl
, SPReg
, VT
);
1587 Chain
= SP
.getValue(1);
1588 unsigned Align
= cast
<ConstantSDNode
>(Tmp3
)->getZExtValue();
1589 unsigned StackAlign
=
1590 DAG
.getSubtarget().getFrameLowering()->getStackAlignment();
1591 Tmp1
= DAG
.getNode(ISD::SUB
, dl
, VT
, SP
, Size
); // Value
1592 if (Align
> StackAlign
)
1593 Tmp1
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp1
,
1594 DAG
.getConstant(-(uint64_t)Align
, dl
, VT
));
1595 Chain
= DAG
.getCopyToReg(Chain
, dl
, SPReg
, Tmp1
); // Output chain
1597 Tmp2
= DAG
.getCALLSEQ_END(Chain
, DAG
.getIntPtrConstant(0, dl
, true),
1598 DAG
.getIntPtrConstant(0, dl
, true), SDValue(), dl
);
1600 Results
.push_back(Tmp1
);
1601 Results
.push_back(Tmp2
);
1604 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1607 /// If the SETCC has been legalized using AND / OR, then the legalized node
1608 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1609 /// will be set to false.
1611 /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1612 /// then the values of LHS and RHS will be swapped, CC will be set to the
1613 /// new condition, and NeedInvert will be set to false.
1615 /// If the SETCC has been legalized using the inverse condcode, then LHS and
1616 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1617 /// will be set to true. The caller must invert the result of the SETCC with
1618 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1619 /// of a true/false result.
1621 /// \returns true if the SetCC has been legalized, false if it hasn't.
1622 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT
, SDValue
&LHS
,
1623 SDValue
&RHS
, SDValue
&CC
,
1626 MVT OpVT
= LHS
.getSimpleValueType();
1627 ISD::CondCode CCCode
= cast
<CondCodeSDNode
>(CC
)->get();
1629 switch (TLI
.getCondCodeAction(CCCode
, OpVT
)) {
1630 default: llvm_unreachable("Unknown condition code action!");
1631 case TargetLowering::Legal
:
1634 case TargetLowering::Expand
: {
1635 ISD::CondCode InvCC
= ISD::getSetCCSwappedOperands(CCCode
);
1636 if (TLI
.isCondCodeLegalOrCustom(InvCC
, OpVT
)) {
1637 std::swap(LHS
, RHS
);
1638 CC
= DAG
.getCondCode(InvCC
);
1641 // Swapping operands didn't work. Try inverting the condition.
1642 bool NeedSwap
= false;
1643 InvCC
= getSetCCInverse(CCCode
, OpVT
.isInteger());
1644 if (!TLI
.isCondCodeLegalOrCustom(InvCC
, OpVT
)) {
1645 // If inverting the condition is not enough, try swapping operands
1647 InvCC
= ISD::getSetCCSwappedOperands(InvCC
);
1650 if (TLI
.isCondCodeLegalOrCustom(InvCC
, OpVT
)) {
1651 CC
= DAG
.getCondCode(InvCC
);
1654 std::swap(LHS
, RHS
);
1658 ISD::CondCode CC1
= ISD::SETCC_INVALID
, CC2
= ISD::SETCC_INVALID
;
1661 default: llvm_unreachable("Don't know how to expand this condition!");
1663 assert(TLI
.isCondCodeLegal(ISD::SETOEQ
, OpVT
)
1664 && "If SETO is expanded, SETOEQ must be legal!");
1665 CC1
= ISD::SETOEQ
; CC2
= ISD::SETOEQ
; Opc
= ISD::AND
; break;
1667 assert(TLI
.isCondCodeLegal(ISD::SETUNE
, OpVT
)
1668 && "If SETUO is expanded, SETUNE must be legal!");
1669 CC1
= ISD::SETUNE
; CC2
= ISD::SETUNE
; Opc
= ISD::OR
; break;
1682 // If we are floating point, assign and break, otherwise fall through.
1683 if (!OpVT
.isInteger()) {
1684 // We can use the 4th bit to tell if we are the unordered
1685 // or ordered version of the opcode.
1686 CC2
= ((unsigned)CCCode
& 0x8U
) ? ISD::SETUO
: ISD::SETO
;
1687 Opc
= ((unsigned)CCCode
& 0x8U
) ? ISD::OR
: ISD::AND
;
1688 CC1
= (ISD::CondCode
)(((int)CCCode
& 0x7) | 0x10);
1691 // Fallthrough if we are unsigned integer.
1699 // If all combinations of inverting the condition and swapping operands
1700 // didn't work then we have no means to expand the condition.
1701 llvm_unreachable("Don't know how to expand this condition!");
1704 SDValue SetCC1
, SetCC2
;
1705 if (CCCode
!= ISD::SETO
&& CCCode
!= ISD::SETUO
) {
1706 // If we aren't the ordered or unorder operation,
1707 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1708 SetCC1
= DAG
.getSetCC(dl
, VT
, LHS
, RHS
, CC1
);
1709 SetCC2
= DAG
.getSetCC(dl
, VT
, LHS
, RHS
, CC2
);
1711 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1712 SetCC1
= DAG
.getSetCC(dl
, VT
, LHS
, LHS
, CC1
);
1713 SetCC2
= DAG
.getSetCC(dl
, VT
, RHS
, RHS
, CC2
);
1715 LHS
= DAG
.getNode(Opc
, dl
, VT
, SetCC1
, SetCC2
);
1724 /// Emit a store/load combination to the stack. This stores
1725 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1726 /// a load from the stack slot to DestVT, extending it if needed.
1727 /// The resultant code need not be legal.
1728 SDValue
SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp
, EVT SlotVT
,
1729 EVT DestVT
, const SDLoc
&dl
) {
1730 return EmitStackConvert(SrcOp
, SlotVT
, DestVT
, dl
, DAG
.getEntryNode());
1733 SDValue
SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp
, EVT SlotVT
,
1734 EVT DestVT
, const SDLoc
&dl
,
1736 // Create the stack frame object.
1737 unsigned SrcAlign
= DAG
.getDataLayout().getPrefTypeAlignment(
1738 SrcOp
.getValueType().getTypeForEVT(*DAG
.getContext()));
1739 SDValue FIPtr
= DAG
.CreateStackTemporary(SlotVT
, SrcAlign
);
1741 FrameIndexSDNode
*StackPtrFI
= cast
<FrameIndexSDNode
>(FIPtr
);
1742 int SPFI
= StackPtrFI
->getIndex();
1743 MachinePointerInfo PtrInfo
=
1744 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), SPFI
);
1746 unsigned SrcSize
= SrcOp
.getValueSizeInBits();
1747 unsigned SlotSize
= SlotVT
.getSizeInBits();
1748 unsigned DestSize
= DestVT
.getSizeInBits();
1749 Type
*DestType
= DestVT
.getTypeForEVT(*DAG
.getContext());
1750 unsigned DestAlign
= DAG
.getDataLayout().getPrefTypeAlignment(DestType
);
1752 // Emit a store to the stack slot. Use a truncstore if the input value is
1753 // later than DestVT.
1756 if (SrcSize
> SlotSize
)
1757 Store
= DAG
.getTruncStore(Chain
, dl
, SrcOp
, FIPtr
, PtrInfo
,
1760 assert(SrcSize
== SlotSize
&& "Invalid store");
1762 DAG
.getStore(Chain
, dl
, SrcOp
, FIPtr
, PtrInfo
, SrcAlign
);
1765 // Result is a load from the stack slot.
1766 if (SlotSize
== DestSize
)
1767 return DAG
.getLoad(DestVT
, dl
, Store
, FIPtr
, PtrInfo
, DestAlign
);
1769 assert(SlotSize
< DestSize
&& "Unknown extension!");
1770 return DAG
.getExtLoad(ISD::EXTLOAD
, dl
, DestVT
, Store
, FIPtr
, PtrInfo
, SlotVT
,
1774 SDValue
SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode
*Node
) {
1776 // Create a vector sized/aligned stack slot, store the value to element #0,
1777 // then load the whole vector back out.
1778 SDValue StackPtr
= DAG
.CreateStackTemporary(Node
->getValueType(0));
1780 FrameIndexSDNode
*StackPtrFI
= cast
<FrameIndexSDNode
>(StackPtr
);
1781 int SPFI
= StackPtrFI
->getIndex();
1783 SDValue Ch
= DAG
.getTruncStore(
1784 DAG
.getEntryNode(), dl
, Node
->getOperand(0), StackPtr
,
1785 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), SPFI
),
1786 Node
->getValueType(0).getVectorElementType());
1788 Node
->getValueType(0), dl
, Ch
, StackPtr
,
1789 MachinePointerInfo::getFixedStack(DAG
.getMachineFunction(), SPFI
));
1793 ExpandBVWithShuffles(SDNode
*Node
, SelectionDAG
&DAG
,
1794 const TargetLowering
&TLI
, SDValue
&Res
) {
1795 unsigned NumElems
= Node
->getNumOperands();
1797 EVT VT
= Node
->getValueType(0);
1799 // Try to group the scalars into pairs, shuffle the pairs together, then
1800 // shuffle the pairs of pairs together, etc. until the vector has
1801 // been built. This will work only if all of the necessary shuffle masks
1804 // We do this in two phases; first to check the legality of the shuffles,
1805 // and next, assuming that all shuffles are legal, to create the new nodes.
1806 for (int Phase
= 0; Phase
< 2; ++Phase
) {
1807 SmallVector
<std::pair
<SDValue
, SmallVector
<int, 16>>, 16> IntermedVals
,
1809 for (unsigned i
= 0; i
< NumElems
; ++i
) {
1810 SDValue V
= Node
->getOperand(i
);
1816 Vec
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, VT
, V
);
1817 IntermedVals
.push_back(std::make_pair(Vec
, SmallVector
<int, 16>(1, i
)));
1820 while (IntermedVals
.size() > 2) {
1821 NewIntermedVals
.clear();
1822 for (unsigned i
= 0, e
= (IntermedVals
.size() & ~1u); i
< e
; i
+= 2) {
1823 // This vector and the next vector are shuffled together (simply to
1824 // append the one to the other).
1825 SmallVector
<int, 16> ShuffleVec(NumElems
, -1);
1827 SmallVector
<int, 16> FinalIndices
;
1828 FinalIndices
.reserve(IntermedVals
[i
].second
.size() +
1829 IntermedVals
[i
+1].second
.size());
1832 for (unsigned j
= 0, f
= IntermedVals
[i
].second
.size(); j
!= f
;
1835 FinalIndices
.push_back(IntermedVals
[i
].second
[j
]);
1837 for (unsigned j
= 0, f
= IntermedVals
[i
+1].second
.size(); j
!= f
;
1839 ShuffleVec
[k
] = NumElems
+ j
;
1840 FinalIndices
.push_back(IntermedVals
[i
+1].second
[j
]);
1845 Shuffle
= DAG
.getVectorShuffle(VT
, dl
, IntermedVals
[i
].first
,
1846 IntermedVals
[i
+1].first
,
1848 else if (!TLI
.isShuffleMaskLegal(ShuffleVec
, VT
))
1850 NewIntermedVals
.push_back(
1851 std::make_pair(Shuffle
, std::move(FinalIndices
)));
1854 // If we had an odd number of defined values, then append the last
1855 // element to the array of new vectors.
1856 if ((IntermedVals
.size() & 1) != 0)
1857 NewIntermedVals
.push_back(IntermedVals
.back());
1859 IntermedVals
.swap(NewIntermedVals
);
1862 assert(IntermedVals
.size() <= 2 && IntermedVals
.size() > 0 &&
1863 "Invalid number of intermediate vectors");
1864 SDValue Vec1
= IntermedVals
[0].first
;
1866 if (IntermedVals
.size() > 1)
1867 Vec2
= IntermedVals
[1].first
;
1869 Vec2
= DAG
.getUNDEF(VT
);
1871 SmallVector
<int, 16> ShuffleVec(NumElems
, -1);
1872 for (unsigned i
= 0, e
= IntermedVals
[0].second
.size(); i
!= e
; ++i
)
1873 ShuffleVec
[IntermedVals
[0].second
[i
]] = i
;
1874 for (unsigned i
= 0, e
= IntermedVals
[1].second
.size(); i
!= e
; ++i
)
1875 ShuffleVec
[IntermedVals
[1].second
[i
]] = NumElems
+ i
;
1878 Res
= DAG
.getVectorShuffle(VT
, dl
, Vec1
, Vec2
, ShuffleVec
);
1879 else if (!TLI
.isShuffleMaskLegal(ShuffleVec
, VT
))
1886 /// Expand a BUILD_VECTOR node on targets that don't
1887 /// support the operation, but do support the resultant vector type.
1888 SDValue
SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode
*Node
) {
1889 unsigned NumElems
= Node
->getNumOperands();
1890 SDValue Value1
, Value2
;
1892 EVT VT
= Node
->getValueType(0);
1893 EVT OpVT
= Node
->getOperand(0).getValueType();
1894 EVT EltVT
= VT
.getVectorElementType();
1896 // If the only non-undef value is the low element, turn this into a
1897 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1898 bool isOnlyLowElement
= true;
1899 bool MoreThanTwoValues
= false;
1900 bool isConstant
= true;
1901 for (unsigned i
= 0; i
< NumElems
; ++i
) {
1902 SDValue V
= Node
->getOperand(i
);
1906 isOnlyLowElement
= false;
1907 if (!isa
<ConstantFPSDNode
>(V
) && !isa
<ConstantSDNode
>(V
))
1910 if (!Value1
.getNode()) {
1912 } else if (!Value2
.getNode()) {
1915 } else if (V
!= Value1
&& V
!= Value2
) {
1916 MoreThanTwoValues
= true;
1920 if (!Value1
.getNode())
1921 return DAG
.getUNDEF(VT
);
1923 if (isOnlyLowElement
)
1924 return DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, VT
, Node
->getOperand(0));
1926 // If all elements are constants, create a load from the constant pool.
1928 SmallVector
<Constant
*, 16> CV
;
1929 for (unsigned i
= 0, e
= NumElems
; i
!= e
; ++i
) {
1930 if (ConstantFPSDNode
*V
=
1931 dyn_cast
<ConstantFPSDNode
>(Node
->getOperand(i
))) {
1932 CV
.push_back(const_cast<ConstantFP
*>(V
->getConstantFPValue()));
1933 } else if (ConstantSDNode
*V
=
1934 dyn_cast
<ConstantSDNode
>(Node
->getOperand(i
))) {
1936 CV
.push_back(const_cast<ConstantInt
*>(V
->getConstantIntValue()));
1938 // If OpVT and EltVT don't match, EltVT is not legal and the
1939 // element values have been promoted/truncated earlier. Undo this;
1940 // we don't want a v16i8 to become a v16i32 for example.
1941 const ConstantInt
*CI
= V
->getConstantIntValue();
1942 CV
.push_back(ConstantInt::get(EltVT
.getTypeForEVT(*DAG
.getContext()),
1943 CI
->getZExtValue()));
1946 assert(Node
->getOperand(i
).isUndef());
1947 Type
*OpNTy
= EltVT
.getTypeForEVT(*DAG
.getContext());
1948 CV
.push_back(UndefValue::get(OpNTy
));
1951 Constant
*CP
= ConstantVector::get(CV
);
1953 DAG
.getConstantPool(CP
, TLI
.getPointerTy(DAG
.getDataLayout()));
1954 unsigned Alignment
= cast
<ConstantPoolSDNode
>(CPIdx
)->getAlignment();
1956 VT
, dl
, DAG
.getEntryNode(), CPIdx
,
1957 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()),
1961 SmallSet
<SDValue
, 16> DefinedValues
;
1962 for (unsigned i
= 0; i
< NumElems
; ++i
) {
1963 if (Node
->getOperand(i
).isUndef())
1965 DefinedValues
.insert(Node
->getOperand(i
));
1968 if (TLI
.shouldExpandBuildVectorWithShuffles(VT
, DefinedValues
.size())) {
1969 if (!MoreThanTwoValues
) {
1970 SmallVector
<int, 8> ShuffleVec(NumElems
, -1);
1971 for (unsigned i
= 0; i
< NumElems
; ++i
) {
1972 SDValue V
= Node
->getOperand(i
);
1975 ShuffleVec
[i
] = V
== Value1
? 0 : NumElems
;
1977 if (TLI
.isShuffleMaskLegal(ShuffleVec
, Node
->getValueType(0))) {
1978 // Get the splatted value into the low element of a vector register.
1979 SDValue Vec1
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, VT
, Value1
);
1981 if (Value2
.getNode())
1982 Vec2
= DAG
.getNode(ISD::SCALAR_TO_VECTOR
, dl
, VT
, Value2
);
1984 Vec2
= DAG
.getUNDEF(VT
);
1986 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1987 return DAG
.getVectorShuffle(VT
, dl
, Vec1
, Vec2
, ShuffleVec
);
1991 if (ExpandBVWithShuffles(Node
, DAG
, TLI
, Res
))
1996 // Otherwise, we can't handle this case efficiently.
1997 return ExpandVectorBuildThroughStack(Node
);
2000 // Expand a node into a call to a libcall. If the result value
2001 // does not fit into a register, return the lo part and set the hi part to the
2002 // by-reg argument. If it does fit into a single register, return the result
2003 // and leave the Hi part unset.
2004 SDValue
SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC
, SDNode
*Node
,
2006 TargetLowering::ArgListTy Args
;
2007 TargetLowering::ArgListEntry Entry
;
2008 for (const SDValue
&Op
: Node
->op_values()) {
2009 EVT ArgVT
= Op
.getValueType();
2010 Type
*ArgTy
= ArgVT
.getTypeForEVT(*DAG
.getContext());
2013 Entry
.IsSExt
= TLI
.shouldSignExtendTypeInLibCall(ArgVT
, isSigned
);
2014 Entry
.IsZExt
= !TLI
.shouldSignExtendTypeInLibCall(ArgVT
, isSigned
);
2015 Args
.push_back(Entry
);
2017 SDValue Callee
= DAG
.getExternalSymbol(TLI
.getLibcallName(LC
),
2018 TLI
.getPointerTy(DAG
.getDataLayout()));
2020 EVT RetVT
= Node
->getValueType(0);
2021 Type
*RetTy
= RetVT
.getTypeForEVT(*DAG
.getContext());
2023 // By default, the input chain to this libcall is the entry node of the
2024 // function. If the libcall is going to be emitted as a tail call then
2025 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2026 // node which is being folded has a non-entry input chain.
2027 SDValue InChain
= DAG
.getEntryNode();
2029 // isTailCall may be true since the callee does not reference caller stack
2030 // frame. Check if it's in the right position and that the return types match.
2031 SDValue TCChain
= InChain
;
2032 const Function
&F
= DAG
.getMachineFunction().getFunction();
2034 TLI
.isInTailCallPosition(DAG
, Node
, TCChain
) &&
2035 (RetTy
== F
.getReturnType() || F
.getReturnType()->isVoidTy());
2039 TargetLowering::CallLoweringInfo
CLI(DAG
);
2040 bool signExtend
= TLI
.shouldSignExtendTypeInLibCall(RetVT
, isSigned
);
2041 CLI
.setDebugLoc(SDLoc(Node
))
2043 .setLibCallee(TLI
.getLibcallCallingConv(LC
), RetTy
, Callee
,
2045 .setTailCall(isTailCall
)
2046 .setSExtResult(signExtend
)
2047 .setZExtResult(!signExtend
)
2048 .setIsPostTypeLegalization(true);
2050 std::pair
<SDValue
, SDValue
> CallInfo
= TLI
.LowerCallTo(CLI
);
2052 if (!CallInfo
.second
.getNode()) {
2053 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG
.getRoot().dump(&DAG
));
2054 // It's a tailcall, return the chain (which is the DAG root).
2055 return DAG
.getRoot();
2058 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo
.first
.dump(&DAG
));
2059 return CallInfo
.first
;
2062 // Expand a node into a call to a libcall. Similar to
2063 // ExpandLibCall except that the first operand is the in-chain.
2064 std::pair
<SDValue
, SDValue
>
2065 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC
,
2068 SDValue InChain
= Node
->getOperand(0);
2070 TargetLowering::ArgListTy Args
;
2071 TargetLowering::ArgListEntry Entry
;
2072 for (unsigned i
= 1, e
= Node
->getNumOperands(); i
!= e
; ++i
) {
2073 EVT ArgVT
= Node
->getOperand(i
).getValueType();
2074 Type
*ArgTy
= ArgVT
.getTypeForEVT(*DAG
.getContext());
2075 Entry
.Node
= Node
->getOperand(i
);
2077 Entry
.IsSExt
= isSigned
;
2078 Entry
.IsZExt
= !isSigned
;
2079 Args
.push_back(Entry
);
2081 SDValue Callee
= DAG
.getExternalSymbol(TLI
.getLibcallName(LC
),
2082 TLI
.getPointerTy(DAG
.getDataLayout()));
2084 Type
*RetTy
= Node
->getValueType(0).getTypeForEVT(*DAG
.getContext());
2086 TargetLowering::CallLoweringInfo
CLI(DAG
);
2087 CLI
.setDebugLoc(SDLoc(Node
))
2089 .setLibCallee(TLI
.getLibcallCallingConv(LC
), RetTy
, Callee
,
2091 .setSExtResult(isSigned
)
2092 .setZExtResult(!isSigned
);
2094 std::pair
<SDValue
, SDValue
> CallInfo
= TLI
.LowerCallTo(CLI
);
2099 SDValue
SelectionDAGLegalize::ExpandFPLibCall(SDNode
* Node
,
2100 RTLIB::Libcall Call_F32
,
2101 RTLIB::Libcall Call_F64
,
2102 RTLIB::Libcall Call_F80
,
2103 RTLIB::Libcall Call_F128
,
2104 RTLIB::Libcall Call_PPCF128
) {
2105 if (Node
->isStrictFPOpcode())
2106 Node
= DAG
.mutateStrictFPToFP(Node
);
2109 switch (Node
->getSimpleValueType(0).SimpleTy
) {
2110 default: llvm_unreachable("Unexpected request for libcall!");
2111 case MVT::f32
: LC
= Call_F32
; break;
2112 case MVT::f64
: LC
= Call_F64
; break;
2113 case MVT::f80
: LC
= Call_F80
; break;
2114 case MVT::f128
: LC
= Call_F128
; break;
2115 case MVT::ppcf128
: LC
= Call_PPCF128
; break;
2117 return ExpandLibCall(LC
, Node
, false);
2120 SDValue
SelectionDAGLegalize::ExpandIntLibCall(SDNode
* Node
, bool isSigned
,
2121 RTLIB::Libcall Call_I8
,
2122 RTLIB::Libcall Call_I16
,
2123 RTLIB::Libcall Call_I32
,
2124 RTLIB::Libcall Call_I64
,
2125 RTLIB::Libcall Call_I128
) {
2127 switch (Node
->getSimpleValueType(0).SimpleTy
) {
2128 default: llvm_unreachable("Unexpected request for libcall!");
2129 case MVT::i8
: LC
= Call_I8
; break;
2130 case MVT::i16
: LC
= Call_I16
; break;
2131 case MVT::i32
: LC
= Call_I32
; break;
2132 case MVT::i64
: LC
= Call_I64
; break;
2133 case MVT::i128
: LC
= Call_I128
; break;
2135 return ExpandLibCall(LC
, Node
, isSigned
);
2138 /// Expand the node to a libcall based on first argument type (for instance
2139 /// lround and its variant).
2140 SDValue
SelectionDAGLegalize::ExpandArgFPLibCall(SDNode
* Node
,
2141 RTLIB::Libcall Call_F32
,
2142 RTLIB::Libcall Call_F64
,
2143 RTLIB::Libcall Call_F80
,
2144 RTLIB::Libcall Call_F128
,
2145 RTLIB::Libcall Call_PPCF128
) {
2147 switch (Node
->getOperand(0).getValueType().getSimpleVT().SimpleTy
) {
2148 default: llvm_unreachable("Unexpected request for libcall!");
2149 case MVT::f32
: LC
= Call_F32
; break;
2150 case MVT::f64
: LC
= Call_F64
; break;
2151 case MVT::f80
: LC
= Call_F80
; break;
2152 case MVT::f128
: LC
= Call_F128
; break;
2153 case MVT::ppcf128
: LC
= Call_PPCF128
; break;
2156 return ExpandLibCall(LC
, Node
, false);
2159 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2161 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode
*Node
,
2162 SmallVectorImpl
<SDValue
> &Results
) {
2163 unsigned Opcode
= Node
->getOpcode();
2164 bool isSigned
= Opcode
== ISD::SDIVREM
;
2167 switch (Node
->getSimpleValueType(0).SimpleTy
) {
2168 default: llvm_unreachable("Unexpected request for libcall!");
2169 case MVT::i8
: LC
= isSigned
? RTLIB::SDIVREM_I8
: RTLIB::UDIVREM_I8
; break;
2170 case MVT::i16
: LC
= isSigned
? RTLIB::SDIVREM_I16
: RTLIB::UDIVREM_I16
; break;
2171 case MVT::i32
: LC
= isSigned
? RTLIB::SDIVREM_I32
: RTLIB::UDIVREM_I32
; break;
2172 case MVT::i64
: LC
= isSigned
? RTLIB::SDIVREM_I64
: RTLIB::UDIVREM_I64
; break;
2173 case MVT::i128
: LC
= isSigned
? RTLIB::SDIVREM_I128
:RTLIB::UDIVREM_I128
; break;
2176 // The input chain to this libcall is the entry node of the function.
2177 // Legalizing the call will automatically add the previous call to the
2179 SDValue InChain
= DAG
.getEntryNode();
2181 EVT RetVT
= Node
->getValueType(0);
2182 Type
*RetTy
= RetVT
.getTypeForEVT(*DAG
.getContext());
2184 TargetLowering::ArgListTy Args
;
2185 TargetLowering::ArgListEntry Entry
;
2186 for (const SDValue
&Op
: Node
->op_values()) {
2187 EVT ArgVT
= Op
.getValueType();
2188 Type
*ArgTy
= ArgVT
.getTypeForEVT(*DAG
.getContext());
2191 Entry
.IsSExt
= isSigned
;
2192 Entry
.IsZExt
= !isSigned
;
2193 Args
.push_back(Entry
);
2196 // Also pass the return address of the remainder.
2197 SDValue FIPtr
= DAG
.CreateStackTemporary(RetVT
);
2199 Entry
.Ty
= RetTy
->getPointerTo();
2200 Entry
.IsSExt
= isSigned
;
2201 Entry
.IsZExt
= !isSigned
;
2202 Args
.push_back(Entry
);
2204 SDValue Callee
= DAG
.getExternalSymbol(TLI
.getLibcallName(LC
),
2205 TLI
.getPointerTy(DAG
.getDataLayout()));
2208 TargetLowering::CallLoweringInfo
CLI(DAG
);
2211 .setLibCallee(TLI
.getLibcallCallingConv(LC
), RetTy
, Callee
,
2213 .setSExtResult(isSigned
)
2214 .setZExtResult(!isSigned
);
2216 std::pair
<SDValue
, SDValue
> CallInfo
= TLI
.LowerCallTo(CLI
);
2218 // Remainder is loaded back from the stack frame.
2220 DAG
.getLoad(RetVT
, dl
, CallInfo
.second
, FIPtr
, MachinePointerInfo());
2221 Results
.push_back(CallInfo
.first
);
2222 Results
.push_back(Rem
);
2225 /// Return true if sincos libcall is available.
2226 static bool isSinCosLibcallAvailable(SDNode
*Node
, const TargetLowering
&TLI
) {
2228 switch (Node
->getSimpleValueType(0).SimpleTy
) {
2229 default: llvm_unreachable("Unexpected request for libcall!");
2230 case MVT::f32
: LC
= RTLIB::SINCOS_F32
; break;
2231 case MVT::f64
: LC
= RTLIB::SINCOS_F64
; break;
2232 case MVT::f80
: LC
= RTLIB::SINCOS_F80
; break;
2233 case MVT::f128
: LC
= RTLIB::SINCOS_F128
; break;
2234 case MVT::ppcf128
: LC
= RTLIB::SINCOS_PPCF128
; break;
2236 return TLI
.getLibcallName(LC
) != nullptr;
2239 /// Only issue sincos libcall if both sin and cos are needed.
2240 static bool useSinCos(SDNode
*Node
) {
2241 unsigned OtherOpcode
= Node
->getOpcode() == ISD::FSIN
2242 ? ISD::FCOS
: ISD::FSIN
;
2244 SDValue Op0
= Node
->getOperand(0);
2245 for (SDNode::use_iterator UI
= Op0
.getNode()->use_begin(),
2246 UE
= Op0
.getNode()->use_end(); UI
!= UE
; ++UI
) {
2250 // The other user might have been turned into sincos already.
2251 if (User
->getOpcode() == OtherOpcode
|| User
->getOpcode() == ISD::FSINCOS
)
2257 /// Issue libcalls to sincos to compute sin / cos pairs.
2259 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode
*Node
,
2260 SmallVectorImpl
<SDValue
> &Results
) {
2262 switch (Node
->getSimpleValueType(0).SimpleTy
) {
2263 default: llvm_unreachable("Unexpected request for libcall!");
2264 case MVT::f32
: LC
= RTLIB::SINCOS_F32
; break;
2265 case MVT::f64
: LC
= RTLIB::SINCOS_F64
; break;
2266 case MVT::f80
: LC
= RTLIB::SINCOS_F80
; break;
2267 case MVT::f128
: LC
= RTLIB::SINCOS_F128
; break;
2268 case MVT::ppcf128
: LC
= RTLIB::SINCOS_PPCF128
; break;
2271 // The input chain to this libcall is the entry node of the function.
2272 // Legalizing the call will automatically add the previous call to the
2274 SDValue InChain
= DAG
.getEntryNode();
2276 EVT RetVT
= Node
->getValueType(0);
2277 Type
*RetTy
= RetVT
.getTypeForEVT(*DAG
.getContext());
2279 TargetLowering::ArgListTy Args
;
2280 TargetLowering::ArgListEntry Entry
;
2282 // Pass the argument.
2283 Entry
.Node
= Node
->getOperand(0);
2285 Entry
.IsSExt
= false;
2286 Entry
.IsZExt
= false;
2287 Args
.push_back(Entry
);
2289 // Pass the return address of sin.
2290 SDValue SinPtr
= DAG
.CreateStackTemporary(RetVT
);
2291 Entry
.Node
= SinPtr
;
2292 Entry
.Ty
= RetTy
->getPointerTo();
2293 Entry
.IsSExt
= false;
2294 Entry
.IsZExt
= false;
2295 Args
.push_back(Entry
);
2297 // Also pass the return address of the cos.
2298 SDValue CosPtr
= DAG
.CreateStackTemporary(RetVT
);
2299 Entry
.Node
= CosPtr
;
2300 Entry
.Ty
= RetTy
->getPointerTo();
2301 Entry
.IsSExt
= false;
2302 Entry
.IsZExt
= false;
2303 Args
.push_back(Entry
);
2305 SDValue Callee
= DAG
.getExternalSymbol(TLI
.getLibcallName(LC
),
2306 TLI
.getPointerTy(DAG
.getDataLayout()));
2309 TargetLowering::CallLoweringInfo
CLI(DAG
);
2310 CLI
.setDebugLoc(dl
).setChain(InChain
).setLibCallee(
2311 TLI
.getLibcallCallingConv(LC
), Type::getVoidTy(*DAG
.getContext()), Callee
,
2314 std::pair
<SDValue
, SDValue
> CallInfo
= TLI
.LowerCallTo(CLI
);
2317 DAG
.getLoad(RetVT
, dl
, CallInfo
.second
, SinPtr
, MachinePointerInfo()));
2319 DAG
.getLoad(RetVT
, dl
, CallInfo
.second
, CosPtr
, MachinePointerInfo()));
2322 /// This function is responsible for legalizing a
2323 /// INT_TO_FP operation of the specified operand when the target requests that
2324 /// we expand it. At this point, we know that the result and operand types are
2325 /// legal for the target.
2326 SDValue
SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned
, SDValue Op0
,
2329 EVT SrcVT
= Op0
.getValueType();
2331 // TODO: Should any fast-math-flags be set for the created nodes?
2332 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2333 if (SrcVT
== MVT::i32
&& TLI
.isTypeLegal(MVT::f64
)) {
2334 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2337 // Get the stack frame index of a 8 byte buffer.
2338 SDValue StackSlot
= DAG
.CreateStackTemporary(MVT::f64
);
2340 // word offset constant for Hi/Lo address computation
2341 SDValue WordOff
= DAG
.getConstant(sizeof(int), dl
,
2342 StackSlot
.getValueType());
2343 // set up Hi and Lo (into buffer) address based on endian
2344 SDValue Hi
= StackSlot
;
2345 SDValue Lo
= DAG
.getNode(ISD::ADD
, dl
, StackSlot
.getValueType(),
2346 StackSlot
, WordOff
);
2347 if (DAG
.getDataLayout().isLittleEndian())
2350 // if signed map to unsigned space
2353 // constant used to invert sign bit (signed to unsigned mapping)
2354 SDValue SignBit
= DAG
.getConstant(0x80000000u
, dl
, MVT::i32
);
2355 Op0Mapped
= DAG
.getNode(ISD::XOR
, dl
, MVT::i32
, Op0
, SignBit
);
2359 // store the lo of the constructed double - based on integer input
2360 SDValue Store1
= DAG
.getStore(DAG
.getEntryNode(), dl
, Op0Mapped
, Lo
,
2361 MachinePointerInfo());
2362 // initial hi portion of constructed double
2363 SDValue InitialHi
= DAG
.getConstant(0x43300000u
, dl
, MVT::i32
);
2364 // store the hi of the constructed double - biased exponent
2366 DAG
.getStore(Store1
, dl
, InitialHi
, Hi
, MachinePointerInfo());
2367 // load the constructed double
2369 DAG
.getLoad(MVT::f64
, dl
, Store2
, StackSlot
, MachinePointerInfo());
2370 // FP constant to bias correct the final result
2371 SDValue Bias
= DAG
.getConstantFP(isSigned
?
2372 BitsToDouble(0x4330000080000000ULL
) :
2373 BitsToDouble(0x4330000000000000ULL
),
2375 // subtract the bias
2376 SDValue Sub
= DAG
.getNode(ISD::FSUB
, dl
, MVT::f64
, Load
, Bias
);
2378 SDValue Result
= DAG
.getFPExtendOrRound(Sub
, dl
, DestVT
);
2381 assert(!isSigned
&& "Legalize cannot Expand SINT_TO_FP for i64 yet");
2382 // Code below here assumes !isSigned without checking again.
2384 SDValue Tmp1
= DAG
.getNode(ISD::SINT_TO_FP
, dl
, DestVT
, Op0
);
2386 SDValue SignSet
= DAG
.getSetCC(dl
, getSetCCResultType(SrcVT
), Op0
,
2387 DAG
.getConstant(0, dl
, SrcVT
), ISD::SETLT
);
2388 SDValue Zero
= DAG
.getIntPtrConstant(0, dl
),
2389 Four
= DAG
.getIntPtrConstant(4, dl
);
2390 SDValue CstOffset
= DAG
.getSelect(dl
, Zero
.getValueType(),
2391 SignSet
, Four
, Zero
);
2393 // If the sign bit of the integer is set, the large number will be treated
2394 // as a negative number. To counteract this, the dynamic code adds an
2395 // offset depending on the data type.
2397 switch (SrcVT
.getSimpleVT().SimpleTy
) {
2398 default: llvm_unreachable("Unsupported integer type!");
2399 case MVT::i8
: FF
= 0x43800000ULL
; break; // 2^8 (as a float)
2400 case MVT::i16
: FF
= 0x47800000ULL
; break; // 2^16 (as a float)
2401 case MVT::i32
: FF
= 0x4F800000ULL
; break; // 2^32 (as a float)
2402 case MVT::i64
: FF
= 0x5F800000ULL
; break; // 2^64 (as a float)
2404 if (DAG
.getDataLayout().isLittleEndian())
2406 Constant
*FudgeFactor
= ConstantInt::get(
2407 Type::getInt64Ty(*DAG
.getContext()), FF
);
2410 DAG
.getConstantPool(FudgeFactor
, TLI
.getPointerTy(DAG
.getDataLayout()));
2411 unsigned Alignment
= cast
<ConstantPoolSDNode
>(CPIdx
)->getAlignment();
2412 CPIdx
= DAG
.getNode(ISD::ADD
, dl
, CPIdx
.getValueType(), CPIdx
, CstOffset
);
2413 Alignment
= std::min(Alignment
, 4u);
2415 if (DestVT
== MVT::f32
)
2416 FudgeInReg
= DAG
.getLoad(
2417 MVT::f32
, dl
, DAG
.getEntryNode(), CPIdx
,
2418 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()),
2421 SDValue Load
= DAG
.getExtLoad(
2422 ISD::EXTLOAD
, dl
, DestVT
, DAG
.getEntryNode(), CPIdx
,
2423 MachinePointerInfo::getConstantPool(DAG
.getMachineFunction()), MVT::f32
,
2425 HandleSDNode
Handle(Load
);
2426 LegalizeOp(Load
.getNode());
2427 FudgeInReg
= Handle
.getValue();
2430 return DAG
.getNode(ISD::FADD
, dl
, DestVT
, Tmp1
, FudgeInReg
);
2433 /// This function is responsible for legalizing a
2434 /// *INT_TO_FP operation of the specified operand when the target requests that
2435 /// we promote it. At this point, we know that the result and operand types are
2436 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2437 /// operation that takes a larger input.
2438 SDValue
SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp
, EVT DestVT
,
2441 // First step, figure out the appropriate *INT_TO_FP operation to use.
2442 EVT NewInTy
= LegalOp
.getValueType();
2444 unsigned OpToUse
= 0;
2446 // Scan for the appropriate larger type to use.
2448 NewInTy
= (MVT::SimpleValueType
)(NewInTy
.getSimpleVT().SimpleTy
+1);
2449 assert(NewInTy
.isInteger() && "Ran out of possibilities!");
2451 // If the target supports SINT_TO_FP of this type, use it.
2452 if (TLI
.isOperationLegalOrCustom(ISD::SINT_TO_FP
, NewInTy
)) {
2453 OpToUse
= ISD::SINT_TO_FP
;
2456 if (isSigned
) continue;
2458 // If the target supports UINT_TO_FP of this type, use it.
2459 if (TLI
.isOperationLegalOrCustom(ISD::UINT_TO_FP
, NewInTy
)) {
2460 OpToUse
= ISD::UINT_TO_FP
;
2464 // Otherwise, try a larger type.
2467 // Okay, we found the operation and type to use. Zero extend our input to the
2468 // desired type then run the operation on it.
2469 return DAG
.getNode(OpToUse
, dl
, DestVT
,
2470 DAG
.getNode(isSigned
? ISD::SIGN_EXTEND
: ISD::ZERO_EXTEND
,
2471 dl
, NewInTy
, LegalOp
));
2474 /// This function is responsible for legalizing a
2475 /// FP_TO_*INT operation of the specified operand when the target requests that
2476 /// we promote it. At this point, we know that the result and operand types are
2477 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2478 /// operation that returns a larger result.
2479 SDValue
SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp
, EVT DestVT
,
2482 // First step, figure out the appropriate FP_TO*INT operation to use.
2483 EVT NewOutTy
= DestVT
;
2485 unsigned OpToUse
= 0;
2487 // Scan for the appropriate larger type to use.
2489 NewOutTy
= (MVT::SimpleValueType
)(NewOutTy
.getSimpleVT().SimpleTy
+1);
2490 assert(NewOutTy
.isInteger() && "Ran out of possibilities!");
2492 // A larger signed type can hold all unsigned values of the requested type,
2493 // so using FP_TO_SINT is valid
2494 if (TLI
.isOperationLegalOrCustom(ISD::FP_TO_SINT
, NewOutTy
)) {
2495 OpToUse
= ISD::FP_TO_SINT
;
2499 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2500 if (!isSigned
&& TLI
.isOperationLegalOrCustom(ISD::FP_TO_UINT
, NewOutTy
)) {
2501 OpToUse
= ISD::FP_TO_UINT
;
2505 // Otherwise, try a larger type.
2508 // Okay, we found the operation and type to use.
2509 SDValue Operation
= DAG
.getNode(OpToUse
, dl
, NewOutTy
, LegalOp
);
2511 // Truncate the result of the extended FP_TO_*INT operation to the desired
2513 return DAG
.getNode(ISD::TRUNCATE
, dl
, DestVT
, Operation
);
2516 /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2517 SDValue
SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op
, const SDLoc
&dl
) {
2518 EVT VT
= Op
.getValueType();
2519 EVT SHVT
= TLI
.getShiftAmountTy(VT
, DAG
.getDataLayout());
2520 unsigned Sz
= VT
.getScalarSizeInBits();
2522 SDValue Tmp
, Tmp2
, Tmp3
;
2524 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2525 // and finally the i1 pairs.
2526 // TODO: We can easily support i4/i2 legal types if any target ever does.
2527 if (Sz
>= 8 && isPowerOf2_32(Sz
)) {
2528 // Create the masks - repeating the pattern every byte.
2529 APInt MaskHi4
= APInt::getSplat(Sz
, APInt(8, 0xF0));
2530 APInt MaskHi2
= APInt::getSplat(Sz
, APInt(8, 0xCC));
2531 APInt MaskHi1
= APInt::getSplat(Sz
, APInt(8, 0xAA));
2532 APInt MaskLo4
= APInt::getSplat(Sz
, APInt(8, 0x0F));
2533 APInt MaskLo2
= APInt::getSplat(Sz
, APInt(8, 0x33));
2534 APInt MaskLo1
= APInt::getSplat(Sz
, APInt(8, 0x55));
2536 // BSWAP if the type is wider than a single byte.
2537 Tmp
= (Sz
> 8 ? DAG
.getNode(ISD::BSWAP
, dl
, VT
, Op
) : Op
);
2539 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2540 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskHi4
, dl
, VT
));
2541 Tmp3
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskLo4
, dl
, VT
));
2542 Tmp2
= DAG
.getNode(ISD::SRL
, dl
, VT
, Tmp2
, DAG
.getConstant(4, dl
, SHVT
));
2543 Tmp3
= DAG
.getNode(ISD::SHL
, dl
, VT
, Tmp3
, DAG
.getConstant(4, dl
, SHVT
));
2544 Tmp
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp2
, Tmp3
);
2546 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2547 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskHi2
, dl
, VT
));
2548 Tmp3
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskLo2
, dl
, VT
));
2549 Tmp2
= DAG
.getNode(ISD::SRL
, dl
, VT
, Tmp2
, DAG
.getConstant(2, dl
, SHVT
));
2550 Tmp3
= DAG
.getNode(ISD::SHL
, dl
, VT
, Tmp3
, DAG
.getConstant(2, dl
, SHVT
));
2551 Tmp
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp2
, Tmp3
);
2553 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2554 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskHi1
, dl
, VT
));
2555 Tmp3
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp
, DAG
.getConstant(MaskLo1
, dl
, VT
));
2556 Tmp2
= DAG
.getNode(ISD::SRL
, dl
, VT
, Tmp2
, DAG
.getConstant(1, dl
, SHVT
));
2557 Tmp3
= DAG
.getNode(ISD::SHL
, dl
, VT
, Tmp3
, DAG
.getConstant(1, dl
, SHVT
));
2558 Tmp
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp2
, Tmp3
);
2562 Tmp
= DAG
.getConstant(0, dl
, VT
);
2563 for (unsigned I
= 0, J
= Sz
-1; I
< Sz
; ++I
, --J
) {
2566 DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(J
- I
, dl
, SHVT
));
2569 DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(I
- J
, dl
, SHVT
));
2573 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp2
, DAG
.getConstant(Shift
, dl
, VT
));
2574 Tmp
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp
, Tmp2
);
2580 /// Open code the operations for BSWAP of the specified operation.
2581 SDValue
SelectionDAGLegalize::ExpandBSWAP(SDValue Op
, const SDLoc
&dl
) {
2582 EVT VT
= Op
.getValueType();
2583 EVT SHVT
= TLI
.getShiftAmountTy(VT
, DAG
.getDataLayout());
2584 SDValue Tmp1
, Tmp2
, Tmp3
, Tmp4
, Tmp5
, Tmp6
, Tmp7
, Tmp8
;
2585 switch (VT
.getSimpleVT().getScalarType().SimpleTy
) {
2586 default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2588 // Use a rotate by 8. This can be further expanded if necessary.
2589 return DAG
.getNode(ISD::ROTL
, dl
, VT
, Op
, DAG
.getConstant(8, dl
, SHVT
));
2591 Tmp4
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(24, dl
, SHVT
));
2592 Tmp3
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(8, dl
, SHVT
));
2593 Tmp2
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(8, dl
, SHVT
));
2594 Tmp1
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(24, dl
, SHVT
));
2595 Tmp3
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp3
,
2596 DAG
.getConstant(0xFF0000, dl
, VT
));
2597 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp2
, DAG
.getConstant(0xFF00, dl
, VT
));
2598 Tmp4
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp4
, Tmp3
);
2599 Tmp2
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp2
, Tmp1
);
2600 return DAG
.getNode(ISD::OR
, dl
, VT
, Tmp4
, Tmp2
);
2602 Tmp8
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(56, dl
, SHVT
));
2603 Tmp7
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(40, dl
, SHVT
));
2604 Tmp6
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(24, dl
, SHVT
));
2605 Tmp5
= DAG
.getNode(ISD::SHL
, dl
, VT
, Op
, DAG
.getConstant(8, dl
, SHVT
));
2606 Tmp4
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(8, dl
, SHVT
));
2607 Tmp3
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(24, dl
, SHVT
));
2608 Tmp2
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(40, dl
, SHVT
));
2609 Tmp1
= DAG
.getNode(ISD::SRL
, dl
, VT
, Op
, DAG
.getConstant(56, dl
, SHVT
));
2610 Tmp7
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp7
,
2611 DAG
.getConstant(255ULL<<48, dl
, VT
));
2612 Tmp6
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp6
,
2613 DAG
.getConstant(255ULL<<40, dl
, VT
));
2614 Tmp5
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp5
,
2615 DAG
.getConstant(255ULL<<32, dl
, VT
));
2616 Tmp4
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp4
,
2617 DAG
.getConstant(255ULL<<24, dl
, VT
));
2618 Tmp3
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp3
,
2619 DAG
.getConstant(255ULL<<16, dl
, VT
));
2620 Tmp2
= DAG
.getNode(ISD::AND
, dl
, VT
, Tmp2
,
2621 DAG
.getConstant(255ULL<<8 , dl
, VT
));
2622 Tmp8
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp8
, Tmp7
);
2623 Tmp6
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp6
, Tmp5
);
2624 Tmp4
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp4
, Tmp3
);
2625 Tmp2
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp2
, Tmp1
);
2626 Tmp8
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp8
, Tmp6
);
2627 Tmp4
= DAG
.getNode(ISD::OR
, dl
, VT
, Tmp4
, Tmp2
);
2628 return DAG
.getNode(ISD::OR
, dl
, VT
, Tmp8
, Tmp4
);
2632 bool SelectionDAGLegalize::ExpandNode(SDNode
*Node
) {
2633 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2634 SmallVector
<SDValue
, 8> Results
;
2636 SDValue Tmp1
, Tmp2
, Tmp3
, Tmp4
;
2638 switch (Node
->getOpcode()) {
2640 if (TLI
.expandABS(Node
, Tmp1
, DAG
))
2641 Results
.push_back(Tmp1
);
2644 if (TLI
.expandCTPOP(Node
, Tmp1
, DAG
))
2645 Results
.push_back(Tmp1
);
2648 case ISD::CTLZ_ZERO_UNDEF
:
2649 if (TLI
.expandCTLZ(Node
, Tmp1
, DAG
))
2650 Results
.push_back(Tmp1
);
2653 case ISD::CTTZ_ZERO_UNDEF
:
2654 if (TLI
.expandCTTZ(Node
, Tmp1
, DAG
))
2655 Results
.push_back(Tmp1
);
2657 case ISD::BITREVERSE
:
2658 Results
.push_back(ExpandBITREVERSE(Node
->getOperand(0), dl
));
2661 Results
.push_back(ExpandBSWAP(Node
->getOperand(0), dl
));
2663 case ISD::FRAMEADDR
:
2664 case ISD::RETURNADDR
:
2665 case ISD::FRAME_TO_ARGS_OFFSET
:
2666 Results
.push_back(DAG
.getConstant(0, dl
, Node
->getValueType(0)));
2668 case ISD::EH_DWARF_CFA
: {
2669 SDValue CfaArg
= DAG
.getSExtOrTrunc(Node
->getOperand(0), dl
,
2670 TLI
.getPointerTy(DAG
.getDataLayout()));
2671 SDValue Offset
= DAG
.getNode(ISD::ADD
, dl
,
2672 CfaArg
.getValueType(),
2673 DAG
.getNode(ISD::FRAME_TO_ARGS_OFFSET
, dl
,
2674 CfaArg
.getValueType()),
2676 SDValue FA
= DAG
.getNode(
2677 ISD::FRAMEADDR
, dl
, TLI
.getPointerTy(DAG
.getDataLayout()),
2678 DAG
.getConstant(0, dl
, TLI
.getPointerTy(DAG
.getDataLayout())));
2679 Results
.push_back(DAG
.getNode(ISD::ADD
, dl
, FA
.getValueType(),
2683 case ISD::FLT_ROUNDS_
:
2684 Results
.push_back(DAG
.getConstant(1, dl
, Node
->getValueType(0)));
2686 case ISD::EH_RETURN
:
2690 case ISD::EH_SJLJ_LONGJMP
:
2691 // If the target didn't expand these, there's nothing to do, so just
2692 // preserve the chain and be done.
2693 Results
.push_back(Node
->getOperand(0));
2695 case ISD::READCYCLECOUNTER
:
2696 // If the target didn't expand this, just return 'zero' and preserve the
2698 Results
.append(Node
->getNumValues() - 1,
2699 DAG
.getConstant(0, dl
, Node
->getValueType(0)));
2700 Results
.push_back(Node
->getOperand(0));
2702 case ISD::EH_SJLJ_SETJMP
:
2703 // If the target didn't expand this, just return 'zero' and preserve the
2705 Results
.push_back(DAG
.getConstant(0, dl
, MVT::i32
));
2706 Results
.push_back(Node
->getOperand(0));
2708 case ISD::ATOMIC_LOAD
: {
2709 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2710 SDValue Zero
= DAG
.getConstant(0, dl
, Node
->getValueType(0));
2711 SDVTList VTs
= DAG
.getVTList(Node
->getValueType(0), MVT::Other
);
2712 SDValue Swap
= DAG
.getAtomicCmpSwap(
2713 ISD::ATOMIC_CMP_SWAP
, dl
, cast
<AtomicSDNode
>(Node
)->getMemoryVT(), VTs
,
2714 Node
->getOperand(0), Node
->getOperand(1), Zero
, Zero
,
2715 cast
<AtomicSDNode
>(Node
)->getMemOperand());
2716 Results
.push_back(Swap
.getValue(0));
2717 Results
.push_back(Swap
.getValue(1));
2720 case ISD::ATOMIC_STORE
: {
2721 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2722 SDValue Swap
= DAG
.getAtomic(ISD::ATOMIC_SWAP
, dl
,
2723 cast
<AtomicSDNode
>(Node
)->getMemoryVT(),
2724 Node
->getOperand(0),
2725 Node
->getOperand(1), Node
->getOperand(2),
2726 cast
<AtomicSDNode
>(Node
)->getMemOperand());
2727 Results
.push_back(Swap
.getValue(1));
2730 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
: {
2731 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2732 // splits out the success value as a comparison. Expanding the resulting
2733 // ATOMIC_CMP_SWAP will produce a libcall.
2734 SDVTList VTs
= DAG
.getVTList(Node
->getValueType(0), MVT::Other
);
2735 SDValue Res
= DAG
.getAtomicCmpSwap(
2736 ISD::ATOMIC_CMP_SWAP
, dl
, cast
<AtomicSDNode
>(Node
)->getMemoryVT(), VTs
,
2737 Node
->getOperand(0), Node
->getOperand(1), Node
->getOperand(2),
2738 Node
->getOperand(3), cast
<MemSDNode
>(Node
)->getMemOperand());
2740 SDValue ExtRes
= Res
;
2742 SDValue RHS
= Node
->getOperand(1);
2744 EVT AtomicType
= cast
<AtomicSDNode
>(Node
)->getMemoryVT();
2745 EVT OuterType
= Node
->getValueType(0);
2746 switch (TLI
.getExtendForAtomicOps()) {
2747 case ISD::SIGN_EXTEND
:
2748 LHS
= DAG
.getNode(ISD::AssertSext
, dl
, OuterType
, Res
,
2749 DAG
.getValueType(AtomicType
));
2750 RHS
= DAG
.getNode(ISD::SIGN_EXTEND_INREG
, dl
, OuterType
,
2751 Node
->getOperand(2), DAG
.getValueType(AtomicType
));
2754 case ISD::ZERO_EXTEND
:
2755 LHS
= DAG
.getNode(ISD::AssertZext
, dl
, OuterType
, Res
,
2756 DAG
.getValueType(AtomicType
));
2757 RHS
= DAG
.getZeroExtendInReg(Node
->getOperand(2), dl
, AtomicType
);
2760 case ISD::ANY_EXTEND
:
2761 LHS
= DAG
.getZeroExtendInReg(Res
, dl
, AtomicType
);
2762 RHS
= DAG
.getZeroExtendInReg(Node
->getOperand(2), dl
, AtomicType
);
2765 llvm_unreachable("Invalid atomic op extension");
2769 DAG
.getSetCC(dl
, Node
->getValueType(1), LHS
, RHS
, ISD::SETEQ
);
2771 Results
.push_back(ExtRes
.getValue(0));
2772 Results
.push_back(Success
);
2773 Results
.push_back(Res
.getValue(1));
2776 case ISD::DYNAMIC_STACKALLOC
:
2777 ExpandDYNAMIC_STACKALLOC(Node
, Results
);
2779 case ISD::MERGE_VALUES
:
2780 for (unsigned i
= 0; i
< Node
->getNumValues(); i
++)
2781 Results
.push_back(Node
->getOperand(i
));
2784 EVT VT
= Node
->getValueType(0);
2786 Results
.push_back(DAG
.getConstant(0, dl
, VT
));
2788 assert(VT
.isFloatingPoint() && "Unknown value type!");
2789 Results
.push_back(DAG
.getConstantFP(0, dl
, VT
));
2793 case ISD::STRICT_FP_ROUND
:
2794 // This expansion does not honor the "strict" properties anyway,
2795 // so prefer falling back to the non-strict operation if legal.
2796 if (TLI
.getStrictFPOperationAction(Node
->getOpcode(),
2797 Node
->getValueType(0))
2798 == TargetLowering::Legal
)
2800 Tmp1
= EmitStackConvert(Node
->getOperand(1),
2801 Node
->getValueType(0),
2802 Node
->getValueType(0), dl
, Node
->getOperand(0));
2803 ReplaceNode(Node
, Tmp1
.getNode());
2804 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2808 Tmp1
= EmitStackConvert(Node
->getOperand(0),
2809 Node
->getValueType(0),
2810 Node
->getValueType(0), dl
);
2811 Results
.push_back(Tmp1
);
2813 case ISD::STRICT_FP_EXTEND
:
2814 // This expansion does not honor the "strict" properties anyway,
2815 // so prefer falling back to the non-strict operation if legal.
2816 if (TLI
.getStrictFPOperationAction(Node
->getOpcode(),
2817 Node
->getValueType(0))
2818 == TargetLowering::Legal
)
2820 Tmp1
= EmitStackConvert(Node
->getOperand(1),
2821 Node
->getOperand(1).getValueType(),
2822 Node
->getValueType(0), dl
, Node
->getOperand(0));
2823 ReplaceNode(Node
, Tmp1
.getNode());
2824 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2826 case ISD::FP_EXTEND
:
2827 Tmp1
= EmitStackConvert(Node
->getOperand(0),
2828 Node
->getOperand(0).getValueType(),
2829 Node
->getValueType(0), dl
);
2830 Results
.push_back(Tmp1
);
2832 case ISD::SIGN_EXTEND_INREG
: {
2833 EVT ExtraVT
= cast
<VTSDNode
>(Node
->getOperand(1))->getVT();
2834 EVT VT
= Node
->getValueType(0);
2836 // An in-register sign-extend of a boolean is a negation:
2837 // 'true' (1) sign-extended is -1.
2838 // 'false' (0) sign-extended is 0.
2839 // However, we must mask the high bits of the source operand because the
2840 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2842 // TODO: Do this for vectors too?
2843 if (ExtraVT
.getSizeInBits() == 1) {
2844 SDValue One
= DAG
.getConstant(1, dl
, VT
);
2845 SDValue And
= DAG
.getNode(ISD::AND
, dl
, VT
, Node
->getOperand(0), One
);
2846 SDValue Zero
= DAG
.getConstant(0, dl
, VT
);
2847 SDValue Neg
= DAG
.getNode(ISD::SUB
, dl
, VT
, Zero
, And
);
2848 Results
.push_back(Neg
);
2852 // NOTE: we could fall back on load/store here too for targets without
2853 // SRA. However, it is doubtful that any exist.
2854 EVT ShiftAmountTy
= TLI
.getShiftAmountTy(VT
, DAG
.getDataLayout());
2855 unsigned BitsDiff
= VT
.getScalarSizeInBits() -
2856 ExtraVT
.getScalarSizeInBits();
2857 SDValue ShiftCst
= DAG
.getConstant(BitsDiff
, dl
, ShiftAmountTy
);
2858 Tmp1
= DAG
.getNode(ISD::SHL
, dl
, Node
->getValueType(0),
2859 Node
->getOperand(0), ShiftCst
);
2860 Tmp1
= DAG
.getNode(ISD::SRA
, dl
, Node
->getValueType(0), Tmp1
, ShiftCst
);
2861 Results
.push_back(Tmp1
);
2864 case ISD::FP_ROUND_INREG
: {
2865 // The only way we can lower this is to turn it into a TRUNCSTORE,
2866 // EXTLOAD pair, targeting a temporary location (a stack slot).
2868 // NOTE: there is a choice here between constantly creating new stack
2869 // slots and always reusing the same one. We currently always create
2870 // new ones, as reuse may inhibit scheduling.
2871 EVT ExtraVT
= cast
<VTSDNode
>(Node
->getOperand(1))->getVT();
2872 Tmp1
= EmitStackConvert(Node
->getOperand(0), ExtraVT
,
2873 Node
->getValueType(0), dl
);
2874 Results
.push_back(Tmp1
);
2877 case ISD::UINT_TO_FP
:
2878 if (TLI
.expandUINT_TO_FP(Node
, Tmp1
, DAG
)) {
2879 Results
.push_back(Tmp1
);
2883 case ISD::SINT_TO_FP
:
2884 Tmp1
= ExpandLegalINT_TO_FP(Node
->getOpcode() == ISD::SINT_TO_FP
,
2885 Node
->getOperand(0), Node
->getValueType(0), dl
);
2886 Results
.push_back(Tmp1
);
2888 case ISD::FP_TO_SINT
:
2889 if (TLI
.expandFP_TO_SINT(Node
, Tmp1
, DAG
))
2890 Results
.push_back(Tmp1
);
2892 case ISD::STRICT_FP_TO_SINT
:
2893 if (TLI
.expandFP_TO_SINT(Node
, Tmp1
, DAG
)) {
2894 ReplaceNode(Node
, Tmp1
.getNode());
2895 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2899 case ISD::FP_TO_UINT
:
2900 if (TLI
.expandFP_TO_UINT(Node
, Tmp1
, Tmp2
, DAG
))
2901 Results
.push_back(Tmp1
);
2903 case ISD::STRICT_FP_TO_UINT
:
2904 if (TLI
.expandFP_TO_UINT(Node
, Tmp1
, Tmp2
, DAG
)) {
2905 // Relink the chain.
2906 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
,1), Tmp2
);
2907 // Replace the new UINT result.
2908 ReplaceNodeWithValue(SDValue(Node
, 0), Tmp1
);
2909 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2914 Results
.push_back(ExpandArgFPLibCall(Node
, RTLIB::LROUND_F32
,
2915 RTLIB::LROUND_F64
, RTLIB::LROUND_F80
,
2917 RTLIB::LROUND_PPCF128
));
2920 Results
.push_back(ExpandArgFPLibCall(Node
, RTLIB::LLROUND_F32
,
2921 RTLIB::LLROUND_F64
, RTLIB::LLROUND_F80
,
2922 RTLIB::LLROUND_F128
,
2923 RTLIB::LLROUND_PPCF128
));
2926 Results
.push_back(ExpandArgFPLibCall(Node
, RTLIB::LRINT_F32
,
2927 RTLIB::LRINT_F64
, RTLIB::LRINT_F80
,
2929 RTLIB::LRINT_PPCF128
));
2932 Results
.push_back(ExpandArgFPLibCall(Node
, RTLIB::LLRINT_F32
,
2933 RTLIB::LLRINT_F64
, RTLIB::LLRINT_F80
,
2935 RTLIB::LLRINT_PPCF128
));
2938 Results
.push_back(DAG
.expandVAArg(Node
));
2939 Results
.push_back(Results
[0].getValue(1));
2942 Results
.push_back(DAG
.expandVACopy(Node
));
2944 case ISD::EXTRACT_VECTOR_ELT
:
2945 if (Node
->getOperand(0).getValueType().getVectorNumElements() == 1)
2946 // This must be an access of the only element. Return it.
2947 Tmp1
= DAG
.getNode(ISD::BITCAST
, dl
, Node
->getValueType(0),
2948 Node
->getOperand(0));
2950 Tmp1
= ExpandExtractFromVectorThroughStack(SDValue(Node
, 0));
2951 Results
.push_back(Tmp1
);
2953 case ISD::EXTRACT_SUBVECTOR
:
2954 Results
.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node
, 0)));
2956 case ISD::INSERT_SUBVECTOR
:
2957 Results
.push_back(ExpandInsertToVectorThroughStack(SDValue(Node
, 0)));
2959 case ISD::CONCAT_VECTORS
:
2960 Results
.push_back(ExpandVectorBuildThroughStack(Node
));
2962 case ISD::SCALAR_TO_VECTOR
:
2963 Results
.push_back(ExpandSCALAR_TO_VECTOR(Node
));
2965 case ISD::INSERT_VECTOR_ELT
:
2966 Results
.push_back(ExpandINSERT_VECTOR_ELT(Node
->getOperand(0),
2967 Node
->getOperand(1),
2968 Node
->getOperand(2), dl
));
2970 case ISD::VECTOR_SHUFFLE
: {
2971 SmallVector
<int, 32> NewMask
;
2972 ArrayRef
<int> Mask
= cast
<ShuffleVectorSDNode
>(Node
)->getMask();
2974 EVT VT
= Node
->getValueType(0);
2975 EVT EltVT
= VT
.getVectorElementType();
2976 SDValue Op0
= Node
->getOperand(0);
2977 SDValue Op1
= Node
->getOperand(1);
2978 if (!TLI
.isTypeLegal(EltVT
)) {
2979 EVT NewEltVT
= TLI
.getTypeToTransformTo(*DAG
.getContext(), EltVT
);
2981 // BUILD_VECTOR operands are allowed to be wider than the element type.
2982 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2984 if (NewEltVT
.bitsLT(EltVT
)) {
2985 // Convert shuffle node.
2986 // If original node was v4i64 and the new EltVT is i32,
2987 // cast operands to v8i32 and re-build the mask.
2989 // Calculate new VT, the size of the new VT should be equal to original.
2991 EVT::getVectorVT(*DAG
.getContext(), NewEltVT
,
2992 VT
.getSizeInBits() / NewEltVT
.getSizeInBits());
2993 assert(NewVT
.bitsEq(VT
));
2995 // cast operands to new VT
2996 Op0
= DAG
.getNode(ISD::BITCAST
, dl
, NewVT
, Op0
);
2997 Op1
= DAG
.getNode(ISD::BITCAST
, dl
, NewVT
, Op1
);
2999 // Convert the shuffle mask
3000 unsigned int factor
=
3001 NewVT
.getVectorNumElements()/VT
.getVectorNumElements();
3003 // EltVT gets smaller
3006 for (unsigned i
= 0; i
< VT
.getVectorNumElements(); ++i
) {
3008 for (unsigned fi
= 0; fi
< factor
; ++fi
)
3009 NewMask
.push_back(Mask
[i
]);
3012 for (unsigned fi
= 0; fi
< factor
; ++fi
)
3013 NewMask
.push_back(Mask
[i
]*factor
+fi
);
3021 unsigned NumElems
= VT
.getVectorNumElements();
3022 SmallVector
<SDValue
, 16> Ops
;
3023 for (unsigned i
= 0; i
!= NumElems
; ++i
) {
3025 Ops
.push_back(DAG
.getUNDEF(EltVT
));
3028 unsigned Idx
= Mask
[i
];
3030 Ops
.push_back(DAG
.getNode(
3031 ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, Op0
,
3032 DAG
.getConstant(Idx
, dl
, TLI
.getVectorIdxTy(DAG
.getDataLayout()))));
3034 Ops
.push_back(DAG
.getNode(
3035 ISD::EXTRACT_VECTOR_ELT
, dl
, EltVT
, Op1
,
3036 DAG
.getConstant(Idx
- NumElems
, dl
,
3037 TLI
.getVectorIdxTy(DAG
.getDataLayout()))));
3040 Tmp1
= DAG
.getBuildVector(VT
, dl
, Ops
);
3041 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3042 Tmp1
= DAG
.getNode(ISD::BITCAST
, dl
, Node
->getValueType(0), Tmp1
);
3043 Results
.push_back(Tmp1
);
3046 case ISD::EXTRACT_ELEMENT
: {
3047 EVT OpTy
= Node
->getOperand(0).getValueType();
3048 if (cast
<ConstantSDNode
>(Node
->getOperand(1))->getZExtValue()) {
3050 Tmp1
= DAG
.getNode(ISD::SRL
, dl
, OpTy
, Node
->getOperand(0),
3051 DAG
.getConstant(OpTy
.getSizeInBits() / 2, dl
,
3052 TLI
.getShiftAmountTy(
3053 Node
->getOperand(0).getValueType(),
3054 DAG
.getDataLayout())));
3055 Tmp1
= DAG
.getNode(ISD::TRUNCATE
, dl
, Node
->getValueType(0), Tmp1
);
3058 Tmp1
= DAG
.getNode(ISD::TRUNCATE
, dl
, Node
->getValueType(0),
3059 Node
->getOperand(0));
3061 Results
.push_back(Tmp1
);
3064 case ISD::STACKSAVE
:
3065 // Expand to CopyFromReg if the target set
3066 // StackPointerRegisterToSaveRestore.
3067 if (unsigned SP
= TLI
.getStackPointerRegisterToSaveRestore()) {
3068 Results
.push_back(DAG
.getCopyFromReg(Node
->getOperand(0), dl
, SP
,
3069 Node
->getValueType(0)));
3070 Results
.push_back(Results
[0].getValue(1));
3072 Results
.push_back(DAG
.getUNDEF(Node
->getValueType(0)));
3073 Results
.push_back(Node
->getOperand(0));
3076 case ISD::STACKRESTORE
:
3077 // Expand to CopyToReg if the target set
3078 // StackPointerRegisterToSaveRestore.
3079 if (unsigned SP
= TLI
.getStackPointerRegisterToSaveRestore()) {
3080 Results
.push_back(DAG
.getCopyToReg(Node
->getOperand(0), dl
, SP
,
3081 Node
->getOperand(1)));
3083 Results
.push_back(Node
->getOperand(0));
3086 case ISD::GET_DYNAMIC_AREA_OFFSET
:
3087 Results
.push_back(DAG
.getConstant(0, dl
, Node
->getValueType(0)));
3088 Results
.push_back(Results
[0].getValue(0));
3090 case ISD::FCOPYSIGN
:
3091 Results
.push_back(ExpandFCOPYSIGN(Node
));
3094 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3095 Tmp1
= DAG
.getConstantFP(-0.0, dl
, Node
->getValueType(0));
3096 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3097 Tmp1
= DAG
.getNode(ISD::FSUB
, dl
, Node
->getValueType(0), Tmp1
,
3098 Node
->getOperand(0));
3099 Results
.push_back(Tmp1
);
3102 Results
.push_back(ExpandFABS(Node
));
3108 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3110 switch (Node
->getOpcode()) {
3111 default: llvm_unreachable("How did we get here?");
3112 case ISD::SMAX
: Pred
= ISD::SETGT
; break;
3113 case ISD::SMIN
: Pred
= ISD::SETLT
; break;
3114 case ISD::UMAX
: Pred
= ISD::SETUGT
; break;
3115 case ISD::UMIN
: Pred
= ISD::SETULT
; break;
3117 Tmp1
= Node
->getOperand(0);
3118 Tmp2
= Node
->getOperand(1);
3119 Tmp1
= DAG
.getSelectCC(dl
, Tmp1
, Tmp2
, Tmp1
, Tmp2
, Pred
);
3120 Results
.push_back(Tmp1
);
3124 case ISD::FMAXNUM
: {
3125 if (SDValue Expanded
= TLI
.expandFMINNUM_FMAXNUM(Node
, DAG
))
3126 Results
.push_back(Expanded
);
3131 EVT VT
= Node
->getValueType(0);
3132 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3133 // fcos which share the same operand and both are used.
3134 if ((TLI
.isOperationLegalOrCustom(ISD::FSINCOS
, VT
) ||
3135 isSinCosLibcallAvailable(Node
, TLI
))
3136 && useSinCos(Node
)) {
3137 SDVTList VTs
= DAG
.getVTList(VT
, VT
);
3138 Tmp1
= DAG
.getNode(ISD::FSINCOS
, dl
, VTs
, Node
->getOperand(0));
3139 if (Node
->getOpcode() == ISD::FCOS
)
3140 Tmp1
= Tmp1
.getValue(1);
3141 Results
.push_back(Tmp1
);
3146 llvm_unreachable("Illegal fmad should never be formed");
3148 case ISD::FP16_TO_FP
:
3149 if (Node
->getValueType(0) != MVT::f32
) {
3150 // We can extend to types bigger than f32 in two steps without changing
3151 // the result. Since "f16 -> f32" is much more commonly available, give
3152 // CodeGen the option of emitting that before resorting to a libcall.
3154 DAG
.getNode(ISD::FP16_TO_FP
, dl
, MVT::f32
, Node
->getOperand(0));
3156 DAG
.getNode(ISD::FP_EXTEND
, dl
, Node
->getValueType(0), Res
));
3159 case ISD::FP_TO_FP16
:
3160 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3161 if (!TLI
.useSoftFloat() && TM
.Options
.UnsafeFPMath
) {
3162 SDValue Op
= Node
->getOperand(0);
3163 MVT SVT
= Op
.getSimpleValueType();
3164 if ((SVT
== MVT::f64
|| SVT
== MVT::f80
) &&
3165 TLI
.isOperationLegalOrCustom(ISD::FP_TO_FP16
, MVT::f32
)) {
3166 // Under fastmath, we can expand this node into a fround followed by
3167 // a float-half conversion.
3168 SDValue FloatVal
= DAG
.getNode(ISD::FP_ROUND
, dl
, MVT::f32
, Op
,
3169 DAG
.getIntPtrConstant(0, dl
));
3171 DAG
.getNode(ISD::FP_TO_FP16
, dl
, Node
->getValueType(0), FloatVal
));
3175 case ISD::ConstantFP
: {
3176 ConstantFPSDNode
*CFP
= cast
<ConstantFPSDNode
>(Node
);
3177 // Check to see if this FP immediate is already legal.
3178 // If this is a legal constant, turn it into a TargetConstantFP node.
3179 if (!TLI
.isFPImmLegal(CFP
->getValueAPF(), Node
->getValueType(0),
3180 DAG
.getMachineFunction().getFunction().hasOptSize()))
3181 Results
.push_back(ExpandConstantFP(CFP
, true));
3184 case ISD::Constant
: {
3185 ConstantSDNode
*CP
= cast
<ConstantSDNode
>(Node
);
3186 Results
.push_back(ExpandConstant(CP
));
3190 EVT VT
= Node
->getValueType(0);
3191 if (TLI
.isOperationLegalOrCustom(ISD::FADD
, VT
) &&
3192 TLI
.isOperationLegalOrCustom(ISD::FNEG
, VT
)) {
3193 const SDNodeFlags Flags
= Node
->getFlags();
3194 Tmp1
= DAG
.getNode(ISD::FNEG
, dl
, VT
, Node
->getOperand(1));
3195 Tmp1
= DAG
.getNode(ISD::FADD
, dl
, VT
, Node
->getOperand(0), Tmp1
, Flags
);
3196 Results
.push_back(Tmp1
);
3201 EVT VT
= Node
->getValueType(0);
3202 assert(TLI
.isOperationLegalOrCustom(ISD::ADD
, VT
) &&
3203 TLI
.isOperationLegalOrCustom(ISD::XOR
, VT
) &&
3204 "Don't know how to expand this subtraction!");
3205 Tmp1
= DAG
.getNode(ISD::XOR
, dl
, VT
, Node
->getOperand(1),
3206 DAG
.getConstant(APInt::getAllOnesValue(VT
.getSizeInBits()), dl
,
3208 Tmp1
= DAG
.getNode(ISD::ADD
, dl
, VT
, Tmp1
, DAG
.getConstant(1, dl
, VT
));
3209 Results
.push_back(DAG
.getNode(ISD::ADD
, dl
, VT
, Node
->getOperand(0), Tmp1
));
3214 EVT VT
= Node
->getValueType(0);
3215 bool isSigned
= Node
->getOpcode() == ISD::SREM
;
3216 unsigned DivOpc
= isSigned
? ISD::SDIV
: ISD::UDIV
;
3217 unsigned DivRemOpc
= isSigned
? ISD::SDIVREM
: ISD::UDIVREM
;
3218 Tmp2
= Node
->getOperand(0);
3219 Tmp3
= Node
->getOperand(1);
3220 if (TLI
.isOperationLegalOrCustom(DivRemOpc
, VT
)) {
3221 SDVTList VTs
= DAG
.getVTList(VT
, VT
);
3222 Tmp1
= DAG
.getNode(DivRemOpc
, dl
, VTs
, Tmp2
, Tmp3
).getValue(1);
3223 Results
.push_back(Tmp1
);
3224 } else if (TLI
.isOperationLegalOrCustom(DivOpc
, VT
)) {
3226 Tmp1
= DAG
.getNode(DivOpc
, dl
, VT
, Tmp2
, Tmp3
);
3227 Tmp1
= DAG
.getNode(ISD::MUL
, dl
, VT
, Tmp1
, Tmp3
);
3228 Tmp1
= DAG
.getNode(ISD::SUB
, dl
, VT
, Tmp2
, Tmp1
);
3229 Results
.push_back(Tmp1
);
3235 bool isSigned
= Node
->getOpcode() == ISD::SDIV
;
3236 unsigned DivRemOpc
= isSigned
? ISD::SDIVREM
: ISD::UDIVREM
;
3237 EVT VT
= Node
->getValueType(0);
3238 if (TLI
.isOperationLegalOrCustom(DivRemOpc
, VT
)) {
3239 SDVTList VTs
= DAG
.getVTList(VT
, VT
);
3240 Tmp1
= DAG
.getNode(DivRemOpc
, dl
, VTs
, Node
->getOperand(0),
3241 Node
->getOperand(1));
3242 Results
.push_back(Tmp1
);
3248 unsigned ExpandOpcode
=
3249 Node
->getOpcode() == ISD::MULHU
? ISD::UMUL_LOHI
: ISD::SMUL_LOHI
;
3250 EVT VT
= Node
->getValueType(0);
3251 SDVTList VTs
= DAG
.getVTList(VT
, VT
);
3253 Tmp1
= DAG
.getNode(ExpandOpcode
, dl
, VTs
, Node
->getOperand(0),
3254 Node
->getOperand(1));
3255 Results
.push_back(Tmp1
.getValue(1));
3258 case ISD::UMUL_LOHI
:
3259 case ISD::SMUL_LOHI
: {
3260 SDValue LHS
= Node
->getOperand(0);
3261 SDValue RHS
= Node
->getOperand(1);
3262 MVT VT
= LHS
.getSimpleValueType();
3263 unsigned MULHOpcode
=
3264 Node
->getOpcode() == ISD::UMUL_LOHI
? ISD::MULHU
: ISD::MULHS
;
3266 if (TLI
.isOperationLegalOrCustom(MULHOpcode
, VT
)) {
3267 Results
.push_back(DAG
.getNode(ISD::MUL
, dl
, VT
, LHS
, RHS
));
3268 Results
.push_back(DAG
.getNode(MULHOpcode
, dl
, VT
, LHS
, RHS
));
3272 SmallVector
<SDValue
, 4> Halves
;
3273 EVT HalfType
= EVT(VT
).getHalfSizedIntegerVT(*DAG
.getContext());
3274 assert(TLI
.isTypeLegal(HalfType
));
3275 if (TLI
.expandMUL_LOHI(Node
->getOpcode(), VT
, Node
, LHS
, RHS
, Halves
,
3277 TargetLowering::MulExpansionKind::Always
)) {
3278 for (unsigned i
= 0; i
< 2; ++i
) {
3279 SDValue Lo
= DAG
.getNode(ISD::ZERO_EXTEND
, dl
, VT
, Halves
[2 * i
]);
3280 SDValue Hi
= DAG
.getNode(ISD::ANY_EXTEND
, dl
, VT
, Halves
[2 * i
+ 1]);
3281 SDValue Shift
= DAG
.getConstant(
3282 HalfType
.getScalarSizeInBits(), dl
,
3283 TLI
.getShiftAmountTy(HalfType
, DAG
.getDataLayout()));
3284 Hi
= DAG
.getNode(ISD::SHL
, dl
, VT
, Hi
, Shift
);
3285 Results
.push_back(DAG
.getNode(ISD::OR
, dl
, VT
, Lo
, Hi
));
3292 EVT VT
= Node
->getValueType(0);
3293 SDVTList VTs
= DAG
.getVTList(VT
, VT
);
3294 // See if multiply or divide can be lowered using two-result operations.
3295 // We just need the low half of the multiply; try both the signed
3296 // and unsigned forms. If the target supports both SMUL_LOHI and
3297 // UMUL_LOHI, form a preference by checking which forms of plain
3298 // MULH it supports.
3299 bool HasSMUL_LOHI
= TLI
.isOperationLegalOrCustom(ISD::SMUL_LOHI
, VT
);
3300 bool HasUMUL_LOHI
= TLI
.isOperationLegalOrCustom(ISD::UMUL_LOHI
, VT
);
3301 bool HasMULHS
= TLI
.isOperationLegalOrCustom(ISD::MULHS
, VT
);
3302 bool HasMULHU
= TLI
.isOperationLegalOrCustom(ISD::MULHU
, VT
);
3303 unsigned OpToUse
= 0;
3304 if (HasSMUL_LOHI
&& !HasMULHS
) {
3305 OpToUse
= ISD::SMUL_LOHI
;
3306 } else if (HasUMUL_LOHI
&& !HasMULHU
) {
3307 OpToUse
= ISD::UMUL_LOHI
;
3308 } else if (HasSMUL_LOHI
) {
3309 OpToUse
= ISD::SMUL_LOHI
;
3310 } else if (HasUMUL_LOHI
) {
3311 OpToUse
= ISD::UMUL_LOHI
;
3314 Results
.push_back(DAG
.getNode(OpToUse
, dl
, VTs
, Node
->getOperand(0),
3315 Node
->getOperand(1)));
3320 EVT HalfType
= VT
.getHalfSizedIntegerVT(*DAG
.getContext());
3321 if (TLI
.isOperationLegalOrCustom(ISD::ZERO_EXTEND
, VT
) &&
3322 TLI
.isOperationLegalOrCustom(ISD::ANY_EXTEND
, VT
) &&
3323 TLI
.isOperationLegalOrCustom(ISD::SHL
, VT
) &&
3324 TLI
.isOperationLegalOrCustom(ISD::OR
, VT
) &&
3325 TLI
.expandMUL(Node
, Lo
, Hi
, HalfType
, DAG
,
3326 TargetLowering::MulExpansionKind::OnlyLegalOrCustom
)) {
3327 Lo
= DAG
.getNode(ISD::ZERO_EXTEND
, dl
, VT
, Lo
);
3328 Hi
= DAG
.getNode(ISD::ANY_EXTEND
, dl
, VT
, Hi
);
3330 DAG
.getConstant(HalfType
.getSizeInBits(), dl
,
3331 TLI
.getShiftAmountTy(HalfType
, DAG
.getDataLayout()));
3332 Hi
= DAG
.getNode(ISD::SHL
, dl
, VT
, Hi
, Shift
);
3333 Results
.push_back(DAG
.getNode(ISD::OR
, dl
, VT
, Lo
, Hi
));
3339 if (TLI
.expandFunnelShift(Node
, Tmp1
, DAG
))
3340 Results
.push_back(Tmp1
);
3344 if (TLI
.expandROT(Node
, Tmp1
, DAG
))
3345 Results
.push_back(Tmp1
);
3351 Results
.push_back(TLI
.expandAddSubSat(Node
, DAG
));
3354 case ISD::SMULFIXSAT
:
3356 Results
.push_back(TLI
.expandFixedPointMul(Node
, DAG
));
3359 case ISD::SUBCARRY
: {
3360 SDValue LHS
= Node
->getOperand(0);
3361 SDValue RHS
= Node
->getOperand(1);
3362 SDValue Carry
= Node
->getOperand(2);
3364 bool IsAdd
= Node
->getOpcode() == ISD::ADDCARRY
;
3366 // Initial add of the 2 operands.
3367 unsigned Op
= IsAdd
? ISD::ADD
: ISD::SUB
;
3368 EVT VT
= LHS
.getValueType();
3369 SDValue Sum
= DAG
.getNode(Op
, dl
, VT
, LHS
, RHS
);
3371 // Initial check for overflow.
3372 EVT CarryType
= Node
->getValueType(1);
3373 EVT SetCCType
= getSetCCResultType(Node
->getValueType(0));
3374 ISD::CondCode CC
= IsAdd
? ISD::SETULT
: ISD::SETUGT
;
3375 SDValue Overflow
= DAG
.getSetCC(dl
, SetCCType
, Sum
, LHS
, CC
);
3377 // Add of the sum and the carry.
3379 DAG
.getZeroExtendInReg(DAG
.getZExtOrTrunc(Carry
, dl
, VT
), dl
, MVT::i1
);
3380 SDValue Sum2
= DAG
.getNode(Op
, dl
, VT
, Sum
, CarryExt
);
3382 // Second check for overflow. If we are adding, we can only overflow if the
3383 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3384 // If we are subtracting, we can only overflow if the initial sum is 0 and
3385 // the carry is set, resulting in a new sum of all 1s.
3386 SDValue Zero
= DAG
.getConstant(0, dl
, VT
);
3388 IsAdd
? DAG
.getSetCC(dl
, SetCCType
, Sum2
, Zero
, ISD::SETEQ
)
3389 : DAG
.getSetCC(dl
, SetCCType
, Sum
, Zero
, ISD::SETEQ
);
3390 Overflow2
= DAG
.getNode(ISD::AND
, dl
, SetCCType
, Overflow2
,
3391 DAG
.getZExtOrTrunc(Carry
, dl
, SetCCType
));
3393 SDValue ResultCarry
=
3394 DAG
.getNode(ISD::OR
, dl
, SetCCType
, Overflow
, Overflow2
);
3396 Results
.push_back(Sum2
);
3397 Results
.push_back(DAG
.getBoolExtOrTrunc(ResultCarry
, dl
, CarryType
, VT
));
3402 SDValue Result
, Overflow
;
3403 TLI
.expandSADDSUBO(Node
, Result
, Overflow
, DAG
);
3404 Results
.push_back(Result
);
3405 Results
.push_back(Overflow
);
3410 SDValue Result
, Overflow
;
3411 TLI
.expandUADDSUBO(Node
, Result
, Overflow
, DAG
);
3412 Results
.push_back(Result
);
3413 Results
.push_back(Overflow
);
3418 SDValue Result
, Overflow
;
3419 if (TLI
.expandMULO(Node
, Result
, Overflow
, DAG
)) {
3420 Results
.push_back(Result
);
3421 Results
.push_back(Overflow
);
3425 case ISD::BUILD_PAIR
: {
3426 EVT PairTy
= Node
->getValueType(0);
3427 Tmp1
= DAG
.getNode(ISD::ZERO_EXTEND
, dl
, PairTy
, Node
->getOperand(0));
3428 Tmp2
= DAG
.getNode(ISD::ANY_EXTEND
, dl
, PairTy
, Node
->getOperand(1));
3430 ISD::SHL
, dl
, PairTy
, Tmp2
,
3431 DAG
.getConstant(PairTy
.getSizeInBits() / 2, dl
,
3432 TLI
.getShiftAmountTy(PairTy
, DAG
.getDataLayout())));
3433 Results
.push_back(DAG
.getNode(ISD::OR
, dl
, PairTy
, Tmp1
, Tmp2
));
3437 Tmp1
= Node
->getOperand(0);
3438 Tmp2
= Node
->getOperand(1);
3439 Tmp3
= Node
->getOperand(2);
3440 if (Tmp1
.getOpcode() == ISD::SETCC
) {
3441 Tmp1
= DAG
.getSelectCC(dl
, Tmp1
.getOperand(0), Tmp1
.getOperand(1),
3443 cast
<CondCodeSDNode
>(Tmp1
.getOperand(2))->get());
3445 Tmp1
= DAG
.getSelectCC(dl
, Tmp1
,
3446 DAG
.getConstant(0, dl
, Tmp1
.getValueType()),
3447 Tmp2
, Tmp3
, ISD::SETNE
);
3449 Tmp1
->setFlags(Node
->getFlags());
3450 Results
.push_back(Tmp1
);
3453 SDValue Chain
= Node
->getOperand(0);
3454 SDValue Table
= Node
->getOperand(1);
3455 SDValue Index
= Node
->getOperand(2);
3457 const DataLayout
&TD
= DAG
.getDataLayout();
3458 EVT PTy
= TLI
.getPointerTy(TD
);
3460 unsigned EntrySize
=
3461 DAG
.getMachineFunction().getJumpTableInfo()->getEntrySize(TD
);
3463 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3464 // This transformation needs to be done here since otherwise the MIPS
3465 // backend will end up emitting a three instruction multiply sequence
3466 // instead of a single shift and MSP430 will call a runtime function.
3467 if (llvm::isPowerOf2_32(EntrySize
))
3468 Index
= DAG
.getNode(
3469 ISD::SHL
, dl
, Index
.getValueType(), Index
,
3470 DAG
.getConstant(llvm::Log2_32(EntrySize
), dl
, Index
.getValueType()));
3472 Index
= DAG
.getNode(ISD::MUL
, dl
, Index
.getValueType(), Index
,
3473 DAG
.getConstant(EntrySize
, dl
, Index
.getValueType()));
3474 SDValue Addr
= DAG
.getNode(ISD::ADD
, dl
, Index
.getValueType(),
3477 EVT MemVT
= EVT::getIntegerVT(*DAG
.getContext(), EntrySize
* 8);
3478 SDValue LD
= DAG
.getExtLoad(
3479 ISD::SEXTLOAD
, dl
, PTy
, Chain
, Addr
,
3480 MachinePointerInfo::getJumpTable(DAG
.getMachineFunction()), MemVT
);
3482 if (TLI
.isJumpTableRelative()) {
3483 // For PIC, the sequence is:
3484 // BRIND(load(Jumptable + index) + RelocBase)
3485 // RelocBase can be JumpTable, GOT or some sort of global base.
3486 Addr
= DAG
.getNode(ISD::ADD
, dl
, PTy
, Addr
,
3487 TLI
.getPICJumpTableRelocBase(Table
, DAG
));
3490 Tmp1
= TLI
.expandIndirectJTBranch(dl
, LD
.getValue(1), Addr
, DAG
);
3491 Results
.push_back(Tmp1
);
3495 // Expand brcond's setcc into its constituent parts and create a BR_CC
3497 Tmp1
= Node
->getOperand(0);
3498 Tmp2
= Node
->getOperand(1);
3499 if (Tmp2
.getOpcode() == ISD::SETCC
) {
3500 Tmp1
= DAG
.getNode(ISD::BR_CC
, dl
, MVT::Other
,
3501 Tmp1
, Tmp2
.getOperand(2),
3502 Tmp2
.getOperand(0), Tmp2
.getOperand(1),
3503 Node
->getOperand(2));
3505 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3506 if (Tmp2
.isUndef() ||
3507 (Tmp2
.getOpcode() == ISD::AND
&&
3508 isa
<ConstantSDNode
>(Tmp2
.getOperand(1)) &&
3509 cast
<ConstantSDNode
>(Tmp2
.getOperand(1))->getZExtValue() == 1))
3512 Tmp3
= DAG
.getNode(ISD::AND
, dl
, Tmp2
.getValueType(), Tmp2
,
3513 DAG
.getConstant(1, dl
, Tmp2
.getValueType()));
3514 Tmp1
= DAG
.getNode(ISD::BR_CC
, dl
, MVT::Other
, Tmp1
,
3515 DAG
.getCondCode(ISD::SETNE
), Tmp3
,
3516 DAG
.getConstant(0, dl
, Tmp3
.getValueType()),
3517 Node
->getOperand(2));
3519 Results
.push_back(Tmp1
);
3522 Tmp1
= Node
->getOperand(0);
3523 Tmp2
= Node
->getOperand(1);
3524 Tmp3
= Node
->getOperand(2);
3525 bool Legalized
= LegalizeSetCCCondCode(Node
->getValueType(0), Tmp1
, Tmp2
,
3526 Tmp3
, NeedInvert
, dl
);
3529 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3530 // condition code, create a new SETCC node.
3532 Tmp1
= DAG
.getNode(ISD::SETCC
, dl
, Node
->getValueType(0),
3533 Tmp1
, Tmp2
, Tmp3
, Node
->getFlags());
3535 // If we expanded the SETCC by inverting the condition code, then wrap
3536 // the existing SETCC in a NOT to restore the intended condition.
3538 Tmp1
= DAG
.getLogicalNOT(dl
, Tmp1
, Tmp1
->getValueType(0));
3540 Results
.push_back(Tmp1
);
3544 // Otherwise, SETCC for the given comparison type must be completely
3545 // illegal; expand it into a SELECT_CC.
3546 EVT VT
= Node
->getValueType(0);
3548 switch (TLI
.getBooleanContents(Tmp1
.getValueType())) {
3549 case TargetLowering::ZeroOrOneBooleanContent
:
3550 case TargetLowering::UndefinedBooleanContent
:
3553 case TargetLowering::ZeroOrNegativeOneBooleanContent
:
3557 Tmp1
= DAG
.getNode(ISD::SELECT_CC
, dl
, VT
, Tmp1
, Tmp2
,
3558 DAG
.getConstant(TrueValue
, dl
, VT
),
3559 DAG
.getConstant(0, dl
, VT
),
3561 Tmp1
->setFlags(Node
->getFlags());
3562 Results
.push_back(Tmp1
);
3565 case ISD::SELECT_CC
: {
3566 Tmp1
= Node
->getOperand(0); // LHS
3567 Tmp2
= Node
->getOperand(1); // RHS
3568 Tmp3
= Node
->getOperand(2); // True
3569 Tmp4
= Node
->getOperand(3); // False
3570 EVT VT
= Node
->getValueType(0);
3571 SDValue CC
= Node
->getOperand(4);
3572 ISD::CondCode CCOp
= cast
<CondCodeSDNode
>(CC
)->get();
3574 if (TLI
.isCondCodeLegalOrCustom(CCOp
, Tmp1
.getSimpleValueType())) {
3575 // If the condition code is legal, then we need to expand this
3576 // node using SETCC and SELECT.
3577 EVT CmpVT
= Tmp1
.getValueType();
3578 assert(!TLI
.isOperationExpand(ISD::SELECT
, VT
) &&
3579 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3581 EVT CCVT
= getSetCCResultType(CmpVT
);
3582 SDValue Cond
= DAG
.getNode(ISD::SETCC
, dl
, CCVT
, Tmp1
, Tmp2
, CC
, Node
->getFlags());
3583 Results
.push_back(DAG
.getSelect(dl
, VT
, Cond
, Tmp3
, Tmp4
));
3587 // SELECT_CC is legal, so the condition code must not be.
3588 bool Legalized
= false;
3589 // Try to legalize by inverting the condition. This is for targets that
3590 // might support an ordered version of a condition, but not the unordered
3591 // version (or vice versa).
3592 ISD::CondCode InvCC
= ISD::getSetCCInverse(CCOp
,
3593 Tmp1
.getValueType().isInteger());
3594 if (TLI
.isCondCodeLegalOrCustom(InvCC
, Tmp1
.getSimpleValueType())) {
3595 // Use the new condition code and swap true and false
3597 Tmp1
= DAG
.getSelectCC(dl
, Tmp1
, Tmp2
, Tmp4
, Tmp3
, InvCC
);
3598 Tmp1
->setFlags(Node
->getFlags());
3600 // If The inverse is not legal, then try to swap the arguments using
3601 // the inverse condition code.
3602 ISD::CondCode SwapInvCC
= ISD::getSetCCSwappedOperands(InvCC
);
3603 if (TLI
.isCondCodeLegalOrCustom(SwapInvCC
, Tmp1
.getSimpleValueType())) {
3604 // The swapped inverse condition is legal, so swap true and false,
3607 Tmp1
= DAG
.getSelectCC(dl
, Tmp2
, Tmp1
, Tmp4
, Tmp3
, SwapInvCC
);
3608 Tmp1
->setFlags(Node
->getFlags());
3613 Legalized
= LegalizeSetCCCondCode(
3614 getSetCCResultType(Tmp1
.getValueType()), Tmp1
, Tmp2
, CC
, NeedInvert
,
3617 assert(Legalized
&& "Can't legalize SELECT_CC with legal condition!");
3619 // If we expanded the SETCC by inverting the condition code, then swap
3620 // the True/False operands to match.
3622 std::swap(Tmp3
, Tmp4
);
3624 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3625 // condition code, create a new SELECT_CC node.
3627 Tmp1
= DAG
.getNode(ISD::SELECT_CC
, dl
, Node
->getValueType(0),
3628 Tmp1
, Tmp2
, Tmp3
, Tmp4
, CC
);
3630 Tmp2
= DAG
.getConstant(0, dl
, Tmp1
.getValueType());
3631 CC
= DAG
.getCondCode(ISD::SETNE
);
3632 Tmp1
= DAG
.getNode(ISD::SELECT_CC
, dl
, Node
->getValueType(0), Tmp1
,
3633 Tmp2
, Tmp3
, Tmp4
, CC
);
3635 Tmp1
->setFlags(Node
->getFlags());
3637 Results
.push_back(Tmp1
);
3641 Tmp1
= Node
->getOperand(0); // Chain
3642 Tmp2
= Node
->getOperand(2); // LHS
3643 Tmp3
= Node
->getOperand(3); // RHS
3644 Tmp4
= Node
->getOperand(1); // CC
3646 bool Legalized
= LegalizeSetCCCondCode(getSetCCResultType(
3647 Tmp2
.getValueType()), Tmp2
, Tmp3
, Tmp4
, NeedInvert
, dl
);
3649 assert(Legalized
&& "Can't legalize BR_CC with legal condition!");
3651 assert(!NeedInvert
&& "Don't know how to invert BR_CC!");
3653 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3655 if (Tmp4
.getNode()) {
3656 Tmp1
= DAG
.getNode(ISD::BR_CC
, dl
, Node
->getValueType(0), Tmp1
,
3657 Tmp4
, Tmp2
, Tmp3
, Node
->getOperand(4));
3659 Tmp3
= DAG
.getConstant(0, dl
, Tmp2
.getValueType());
3660 Tmp4
= DAG
.getCondCode(ISD::SETNE
);
3661 Tmp1
= DAG
.getNode(ISD::BR_CC
, dl
, Node
->getValueType(0), Tmp1
, Tmp4
,
3662 Tmp2
, Tmp3
, Node
->getOperand(4));
3664 Results
.push_back(Tmp1
);
3667 case ISD::BUILD_VECTOR
:
3668 Results
.push_back(ExpandBUILD_VECTOR(Node
));
3673 // Scalarize vector SRA/SRL/SHL.
3674 EVT VT
= Node
->getValueType(0);
3675 assert(VT
.isVector() && "Unable to legalize non-vector shift");
3676 assert(TLI
.isTypeLegal(VT
.getScalarType())&& "Element type must be legal");
3677 unsigned NumElem
= VT
.getVectorNumElements();
3679 SmallVector
<SDValue
, 8> Scalars
;
3680 for (unsigned Idx
= 0; Idx
< NumElem
; Idx
++) {
3681 SDValue Ex
= DAG
.getNode(
3682 ISD::EXTRACT_VECTOR_ELT
, dl
, VT
.getScalarType(), Node
->getOperand(0),
3683 DAG
.getConstant(Idx
, dl
, TLI
.getVectorIdxTy(DAG
.getDataLayout())));
3684 SDValue Sh
= DAG
.getNode(
3685 ISD::EXTRACT_VECTOR_ELT
, dl
, VT
.getScalarType(), Node
->getOperand(1),
3686 DAG
.getConstant(Idx
, dl
, TLI
.getVectorIdxTy(DAG
.getDataLayout())));
3687 Scalars
.push_back(DAG
.getNode(Node
->getOpcode(), dl
,
3688 VT
.getScalarType(), Ex
, Sh
));
3691 SDValue Result
= DAG
.getBuildVector(Node
->getValueType(0), dl
, Scalars
);
3692 ReplaceNode(SDValue(Node
, 0), Result
);
3695 case ISD::VECREDUCE_FADD
:
3696 case ISD::VECREDUCE_FMUL
:
3697 case ISD::VECREDUCE_ADD
:
3698 case ISD::VECREDUCE_MUL
:
3699 case ISD::VECREDUCE_AND
:
3700 case ISD::VECREDUCE_OR
:
3701 case ISD::VECREDUCE_XOR
:
3702 case ISD::VECREDUCE_SMAX
:
3703 case ISD::VECREDUCE_SMIN
:
3704 case ISD::VECREDUCE_UMAX
:
3705 case ISD::VECREDUCE_UMIN
:
3706 case ISD::VECREDUCE_FMAX
:
3707 case ISD::VECREDUCE_FMIN
:
3708 Results
.push_back(TLI
.expandVecReduce(Node
, DAG
));
3710 case ISD::GLOBAL_OFFSET_TABLE
:
3711 case ISD::GlobalAddress
:
3712 case ISD::GlobalTLSAddress
:
3713 case ISD::ExternalSymbol
:
3714 case ISD::ConstantPool
:
3715 case ISD::JumpTable
:
3716 case ISD::INTRINSIC_W_CHAIN
:
3717 case ISD::INTRINSIC_WO_CHAIN
:
3718 case ISD::INTRINSIC_VOID
:
3719 // FIXME: Custom lowering for these operations shouldn't return null!
3723 if (Results
.empty() && Node
->isStrictFPOpcode()) {
3724 // FIXME: We were asked to expand a strict floating-point operation,
3725 // but there is currently no expansion implemented that would preserve
3726 // the "strict" properties. For now, we just fall back to the non-strict
3727 // version if that is legal on the target. The actual mutation of the
3728 // operation will happen in SelectionDAGISel::DoInstructionSelection.
3729 if (TLI
.getStrictFPOperationAction(Node
->getOpcode(),
3730 Node
->getValueType(0))
3731 == TargetLowering::Legal
)
3735 // Replace the original node with the legalized result.
3736 if (Results
.empty()) {
3737 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3741 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3742 ReplaceNode(Node
, Results
.data());
3746 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode
*Node
) {
3747 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3748 SmallVector
<SDValue
, 8> Results
;
3750 // FIXME: Check flags on the node to see if we can use a finite call.
3751 bool CanUseFiniteLibCall
= TM
.Options
.NoInfsFPMath
&& TM
.Options
.NoNaNsFPMath
;
3752 unsigned Opc
= Node
->getOpcode();
3754 case ISD::ATOMIC_FENCE
: {
3755 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3756 // FIXME: handle "fence singlethread" more efficiently.
3757 TargetLowering::ArgListTy Args
;
3759 TargetLowering::CallLoweringInfo
CLI(DAG
);
3761 .setChain(Node
->getOperand(0))
3763 CallingConv::C
, Type::getVoidTy(*DAG
.getContext()),
3764 DAG
.getExternalSymbol("__sync_synchronize",
3765 TLI
.getPointerTy(DAG
.getDataLayout())),
3768 std::pair
<SDValue
, SDValue
> CallResult
= TLI
.LowerCallTo(CLI
);
3770 Results
.push_back(CallResult
.second
);
3773 // By default, atomic intrinsics are marked Legal and lowered. Targets
3774 // which don't support them directly, however, may want libcalls, in which
3775 // case they mark them Expand, and we get here.
3776 case ISD::ATOMIC_SWAP
:
3777 case ISD::ATOMIC_LOAD_ADD
:
3778 case ISD::ATOMIC_LOAD_SUB
:
3779 case ISD::ATOMIC_LOAD_AND
:
3780 case ISD::ATOMIC_LOAD_CLR
:
3781 case ISD::ATOMIC_LOAD_OR
:
3782 case ISD::ATOMIC_LOAD_XOR
:
3783 case ISD::ATOMIC_LOAD_NAND
:
3784 case ISD::ATOMIC_LOAD_MIN
:
3785 case ISD::ATOMIC_LOAD_MAX
:
3786 case ISD::ATOMIC_LOAD_UMIN
:
3787 case ISD::ATOMIC_LOAD_UMAX
:
3788 case ISD::ATOMIC_CMP_SWAP
: {
3789 MVT VT
= cast
<AtomicSDNode
>(Node
)->getMemoryVT().getSimpleVT();
3790 RTLIB::Libcall LC
= RTLIB::getSYNC(Opc
, VT
);
3791 assert(LC
!= RTLIB::UNKNOWN_LIBCALL
&& "Unexpected atomic op or value type!");
3793 std::pair
<SDValue
, SDValue
> Tmp
= ExpandChainLibCall(LC
, Node
, false);
3794 Results
.push_back(Tmp
.first
);
3795 Results
.push_back(Tmp
.second
);
3799 // If this operation is not supported, lower it to 'abort()' call
3800 TargetLowering::ArgListTy Args
;
3801 TargetLowering::CallLoweringInfo
CLI(DAG
);
3803 .setChain(Node
->getOperand(0))
3804 .setLibCallee(CallingConv::C
, Type::getVoidTy(*DAG
.getContext()),
3805 DAG
.getExternalSymbol(
3806 "abort", TLI
.getPointerTy(DAG
.getDataLayout())),
3808 std::pair
<SDValue
, SDValue
> CallResult
= TLI
.LowerCallTo(CLI
);
3810 Results
.push_back(CallResult
.second
);
3814 case ISD::STRICT_FMINNUM
:
3815 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::FMIN_F32
, RTLIB::FMIN_F64
,
3816 RTLIB::FMIN_F80
, RTLIB::FMIN_F128
,
3817 RTLIB::FMIN_PPCF128
));
3820 case ISD::STRICT_FMAXNUM
:
3821 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::FMAX_F32
, RTLIB::FMAX_F64
,
3822 RTLIB::FMAX_F80
, RTLIB::FMAX_F128
,
3823 RTLIB::FMAX_PPCF128
));
3826 case ISD::STRICT_FSQRT
:
3827 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::SQRT_F32
, RTLIB::SQRT_F64
,
3828 RTLIB::SQRT_F80
, RTLIB::SQRT_F128
,
3829 RTLIB::SQRT_PPCF128
));
3832 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::CBRT_F32
, RTLIB::CBRT_F64
,
3833 RTLIB::CBRT_F80
, RTLIB::CBRT_F128
,
3834 RTLIB::CBRT_PPCF128
));
3837 case ISD::STRICT_FSIN
:
3838 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::SIN_F32
, RTLIB::SIN_F64
,
3839 RTLIB::SIN_F80
, RTLIB::SIN_F128
,
3840 RTLIB::SIN_PPCF128
));
3843 case ISD::STRICT_FCOS
:
3844 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::COS_F32
, RTLIB::COS_F64
,
3845 RTLIB::COS_F80
, RTLIB::COS_F128
,
3846 RTLIB::COS_PPCF128
));
3849 // Expand into sincos libcall.
3850 ExpandSinCosLibCall(Node
, Results
);
3853 case ISD::STRICT_FLOG
:
3854 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_log_finite
))
3855 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG_FINITE_F32
,
3856 RTLIB::LOG_FINITE_F64
,
3857 RTLIB::LOG_FINITE_F80
,
3858 RTLIB::LOG_FINITE_F128
,
3859 RTLIB::LOG_FINITE_PPCF128
));
3861 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG_F32
, RTLIB::LOG_F64
,
3862 RTLIB::LOG_F80
, RTLIB::LOG_F128
,
3863 RTLIB::LOG_PPCF128
));
3866 case ISD::STRICT_FLOG2
:
3867 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_log2_finite
))
3868 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG2_FINITE_F32
,
3869 RTLIB::LOG2_FINITE_F64
,
3870 RTLIB::LOG2_FINITE_F80
,
3871 RTLIB::LOG2_FINITE_F128
,
3872 RTLIB::LOG2_FINITE_PPCF128
));
3874 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG2_F32
, RTLIB::LOG2_F64
,
3875 RTLIB::LOG2_F80
, RTLIB::LOG2_F128
,
3876 RTLIB::LOG2_PPCF128
));
3879 case ISD::STRICT_FLOG10
:
3880 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_log10_finite
))
3881 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG10_FINITE_F32
,
3882 RTLIB::LOG10_FINITE_F64
,
3883 RTLIB::LOG10_FINITE_F80
,
3884 RTLIB::LOG10_FINITE_F128
,
3885 RTLIB::LOG10_FINITE_PPCF128
));
3887 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::LOG10_F32
, RTLIB::LOG10_F64
,
3888 RTLIB::LOG10_F80
, RTLIB::LOG10_F128
,
3889 RTLIB::LOG10_PPCF128
));
3892 case ISD::STRICT_FEXP
:
3893 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_exp_finite
))
3894 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::EXP_FINITE_F32
,
3895 RTLIB::EXP_FINITE_F64
,
3896 RTLIB::EXP_FINITE_F80
,
3897 RTLIB::EXP_FINITE_F128
,
3898 RTLIB::EXP_FINITE_PPCF128
));
3900 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::EXP_F32
, RTLIB::EXP_F64
,
3901 RTLIB::EXP_F80
, RTLIB::EXP_F128
,
3902 RTLIB::EXP_PPCF128
));
3905 case ISD::STRICT_FEXP2
:
3906 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_exp2_finite
))
3907 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::EXP2_FINITE_F32
,
3908 RTLIB::EXP2_FINITE_F64
,
3909 RTLIB::EXP2_FINITE_F80
,
3910 RTLIB::EXP2_FINITE_F128
,
3911 RTLIB::EXP2_FINITE_PPCF128
));
3913 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::EXP2_F32
, RTLIB::EXP2_F64
,
3914 RTLIB::EXP2_F80
, RTLIB::EXP2_F128
,
3915 RTLIB::EXP2_PPCF128
));
3918 case ISD::STRICT_FTRUNC
:
3919 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::TRUNC_F32
, RTLIB::TRUNC_F64
,
3920 RTLIB::TRUNC_F80
, RTLIB::TRUNC_F128
,
3921 RTLIB::TRUNC_PPCF128
));
3924 case ISD::STRICT_FFLOOR
:
3925 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::FLOOR_F32
, RTLIB::FLOOR_F64
,
3926 RTLIB::FLOOR_F80
, RTLIB::FLOOR_F128
,
3927 RTLIB::FLOOR_PPCF128
));
3930 case ISD::STRICT_FCEIL
:
3931 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::CEIL_F32
, RTLIB::CEIL_F64
,
3932 RTLIB::CEIL_F80
, RTLIB::CEIL_F128
,
3933 RTLIB::CEIL_PPCF128
));
3936 case ISD::STRICT_FRINT
:
3937 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::RINT_F32
, RTLIB::RINT_F64
,
3938 RTLIB::RINT_F80
, RTLIB::RINT_F128
,
3939 RTLIB::RINT_PPCF128
));
3941 case ISD::FNEARBYINT
:
3942 case ISD::STRICT_FNEARBYINT
:
3943 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::NEARBYINT_F32
,
3944 RTLIB::NEARBYINT_F64
,
3945 RTLIB::NEARBYINT_F80
,
3946 RTLIB::NEARBYINT_F128
,
3947 RTLIB::NEARBYINT_PPCF128
));
3950 case ISD::STRICT_FROUND
:
3951 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::ROUND_F32
,
3955 RTLIB::ROUND_PPCF128
));
3958 case ISD::STRICT_FPOWI
:
3959 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::POWI_F32
, RTLIB::POWI_F64
,
3960 RTLIB::POWI_F80
, RTLIB::POWI_F128
,
3961 RTLIB::POWI_PPCF128
));
3964 case ISD::STRICT_FPOW
:
3965 if (CanUseFiniteLibCall
&& DAG
.getLibInfo().has(LibFunc_pow_finite
))
3966 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::POW_FINITE_F32
,
3967 RTLIB::POW_FINITE_F64
,
3968 RTLIB::POW_FINITE_F80
,
3969 RTLIB::POW_FINITE_F128
,
3970 RTLIB::POW_FINITE_PPCF128
));
3972 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::POW_F32
, RTLIB::POW_F64
,
3973 RTLIB::POW_F80
, RTLIB::POW_F128
,
3974 RTLIB::POW_PPCF128
));
3977 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::DIV_F32
, RTLIB::DIV_F64
,
3978 RTLIB::DIV_F80
, RTLIB::DIV_F128
,
3979 RTLIB::DIV_PPCF128
));
3982 case ISD::STRICT_FREM
:
3983 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::REM_F32
, RTLIB::REM_F64
,
3984 RTLIB::REM_F80
, RTLIB::REM_F128
,
3985 RTLIB::REM_PPCF128
));
3988 case ISD::STRICT_FMA
:
3989 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::FMA_F32
, RTLIB::FMA_F64
,
3990 RTLIB::FMA_F80
, RTLIB::FMA_F128
,
3991 RTLIB::FMA_PPCF128
));
3994 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::ADD_F32
, RTLIB::ADD_F64
,
3995 RTLIB::ADD_F80
, RTLIB::ADD_F128
,
3996 RTLIB::ADD_PPCF128
));
3999 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::MUL_F32
, RTLIB::MUL_F64
,
4000 RTLIB::MUL_F80
, RTLIB::MUL_F128
,
4001 RTLIB::MUL_PPCF128
));
4003 case ISD::FP16_TO_FP
:
4004 if (Node
->getValueType(0) == MVT::f32
) {
4005 Results
.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32
, Node
, false));
4008 case ISD::FP_TO_FP16
: {
4010 RTLIB::getFPROUND(Node
->getOperand(0).getValueType(), MVT::f16
);
4011 assert(LC
!= RTLIB::UNKNOWN_LIBCALL
&& "Unable to expand fp_to_fp16");
4012 Results
.push_back(ExpandLibCall(LC
, Node
, false));
4016 Results
.push_back(ExpandFPLibCall(Node
, RTLIB::SUB_F32
, RTLIB::SUB_F64
,
4017 RTLIB::SUB_F80
, RTLIB::SUB_F128
,
4018 RTLIB::SUB_PPCF128
));
4021 Results
.push_back(ExpandIntLibCall(Node
, true,
4023 RTLIB::SREM_I16
, RTLIB::SREM_I32
,
4024 RTLIB::SREM_I64
, RTLIB::SREM_I128
));
4027 Results
.push_back(ExpandIntLibCall(Node
, false,
4029 RTLIB::UREM_I16
, RTLIB::UREM_I32
,
4030 RTLIB::UREM_I64
, RTLIB::UREM_I128
));
4033 Results
.push_back(ExpandIntLibCall(Node
, true,
4035 RTLIB::SDIV_I16
, RTLIB::SDIV_I32
,
4036 RTLIB::SDIV_I64
, RTLIB::SDIV_I128
));
4039 Results
.push_back(ExpandIntLibCall(Node
, false,
4041 RTLIB::UDIV_I16
, RTLIB::UDIV_I32
,
4042 RTLIB::UDIV_I64
, RTLIB::UDIV_I128
));
4046 // Expand into divrem libcall
4047 ExpandDivRemLibCall(Node
, Results
);
4050 Results
.push_back(ExpandIntLibCall(Node
, false,
4052 RTLIB::MUL_I16
, RTLIB::MUL_I32
,
4053 RTLIB::MUL_I64
, RTLIB::MUL_I128
));
4055 case ISD::CTLZ_ZERO_UNDEF
:
4056 switch (Node
->getSimpleValueType(0).SimpleTy
) {
4058 llvm_unreachable("LibCall explicitly requested, but not available");
4060 Results
.push_back(ExpandLibCall(RTLIB::CTLZ_I32
, Node
, false));
4063 Results
.push_back(ExpandLibCall(RTLIB::CTLZ_I64
, Node
, false));
4066 Results
.push_back(ExpandLibCall(RTLIB::CTLZ_I128
, Node
, false));
4072 // Replace the original node with the legalized result.
4073 if (!Results
.empty()) {
4074 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4075 ReplaceNode(Node
, Results
.data());
4077 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4080 // Determine the vector type to use in place of an original scalar element when
4081 // promoting equally sized vectors.
4082 static MVT
getPromotedVectorElementType(const TargetLowering
&TLI
,
4083 MVT EltVT
, MVT NewEltVT
) {
4084 unsigned OldEltsPerNewElt
= EltVT
.getSizeInBits() / NewEltVT
.getSizeInBits();
4085 MVT MidVT
= MVT::getVectorVT(NewEltVT
, OldEltsPerNewElt
);
4086 assert(TLI
.isTypeLegal(MidVT
) && "unexpected");
4090 void SelectionDAGLegalize::PromoteNode(SDNode
*Node
) {
4091 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4092 SmallVector
<SDValue
, 8> Results
;
4093 MVT OVT
= Node
->getSimpleValueType(0);
4094 if (Node
->getOpcode() == ISD::UINT_TO_FP
||
4095 Node
->getOpcode() == ISD::SINT_TO_FP
||
4096 Node
->getOpcode() == ISD::SETCC
||
4097 Node
->getOpcode() == ISD::EXTRACT_VECTOR_ELT
||
4098 Node
->getOpcode() == ISD::INSERT_VECTOR_ELT
) {
4099 OVT
= Node
->getOperand(0).getSimpleValueType();
4101 if (Node
->getOpcode() == ISD::BR_CC
)
4102 OVT
= Node
->getOperand(2).getSimpleValueType();
4103 MVT NVT
= TLI
.getTypeToPromoteTo(Node
->getOpcode(), OVT
);
4105 SDValue Tmp1
, Tmp2
, Tmp3
;
4106 switch (Node
->getOpcode()) {
4108 case ISD::CTTZ_ZERO_UNDEF
:
4110 case ISD::CTLZ_ZERO_UNDEF
:
4112 // Zero extend the argument.
4113 Tmp1
= DAG
.getNode(ISD::ZERO_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4114 if (Node
->getOpcode() == ISD::CTTZ
) {
4115 // The count is the same in the promoted type except if the original
4116 // value was zero. This can be handled by setting the bit just off
4117 // the top of the original type.
4118 auto TopBit
= APInt::getOneBitSet(NVT
.getSizeInBits(),
4119 OVT
.getSizeInBits());
4120 Tmp1
= DAG
.getNode(ISD::OR
, dl
, NVT
, Tmp1
,
4121 DAG
.getConstant(TopBit
, dl
, NVT
));
4123 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4124 // already the correct result.
4125 Tmp1
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
);
4126 if (Node
->getOpcode() == ISD::CTLZ
||
4127 Node
->getOpcode() == ISD::CTLZ_ZERO_UNDEF
) {
4128 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4129 Tmp1
= DAG
.getNode(ISD::SUB
, dl
, NVT
, Tmp1
,
4130 DAG
.getConstant(NVT
.getSizeInBits() -
4131 OVT
.getSizeInBits(), dl
, NVT
));
4133 Results
.push_back(DAG
.getNode(ISD::TRUNCATE
, dl
, OVT
, Tmp1
));
4135 case ISD::BITREVERSE
:
4137 unsigned DiffBits
= NVT
.getSizeInBits() - OVT
.getSizeInBits();
4138 Tmp1
= DAG
.getNode(ISD::ZERO_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4139 Tmp1
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
);
4141 ISD::SRL
, dl
, NVT
, Tmp1
,
4142 DAG
.getConstant(DiffBits
, dl
,
4143 TLI
.getShiftAmountTy(NVT
, DAG
.getDataLayout())));
4145 Results
.push_back(DAG
.getNode(ISD::TRUNCATE
, dl
, OVT
, Tmp1
));
4148 case ISD::FP_TO_UINT
:
4149 case ISD::FP_TO_SINT
:
4150 Tmp1
= PromoteLegalFP_TO_INT(Node
->getOperand(0), Node
->getValueType(0),
4151 Node
->getOpcode() == ISD::FP_TO_SINT
, dl
);
4152 Results
.push_back(Tmp1
);
4154 case ISD::UINT_TO_FP
:
4155 case ISD::SINT_TO_FP
:
4156 Tmp1
= PromoteLegalINT_TO_FP(Node
->getOperand(0), Node
->getValueType(0),
4157 Node
->getOpcode() == ISD::SINT_TO_FP
, dl
);
4158 Results
.push_back(Tmp1
);
4161 SDValue Chain
= Node
->getOperand(0); // Get the chain.
4162 SDValue Ptr
= Node
->getOperand(1); // Get the pointer.
4165 if (OVT
.isVector()) {
4166 TruncOp
= ISD::BITCAST
;
4168 assert(OVT
.isInteger()
4169 && "VAARG promotion is supported only for vectors or integer types");
4170 TruncOp
= ISD::TRUNCATE
;
4173 // Perform the larger operation, then convert back
4174 Tmp1
= DAG
.getVAArg(NVT
, dl
, Chain
, Ptr
, Node
->getOperand(2),
4175 Node
->getConstantOperandVal(3));
4176 Chain
= Tmp1
.getValue(1);
4178 Tmp2
= DAG
.getNode(TruncOp
, dl
, OVT
, Tmp1
);
4180 // Modified the chain result - switch anything that used the old chain to
4182 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 0), Tmp2
);
4183 DAG
.ReplaceAllUsesOfValueWith(SDValue(Node
, 1), Chain
);
4185 UpdatedNodes
->insert(Tmp2
.getNode());
4186 UpdatedNodes
->insert(Chain
.getNode());
4199 unsigned ExtOp
, TruncOp
;
4200 if (OVT
.isVector()) {
4201 ExtOp
= ISD::BITCAST
;
4202 TruncOp
= ISD::BITCAST
;
4204 assert(OVT
.isInteger() && "Cannot promote logic operation");
4206 switch (Node
->getOpcode()) {
4208 ExtOp
= ISD::ANY_EXTEND
;
4212 ExtOp
= ISD::SIGN_EXTEND
;
4216 ExtOp
= ISD::ZERO_EXTEND
;
4219 TruncOp
= ISD::TRUNCATE
;
4221 // Promote each of the values to the new type.
4222 Tmp1
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(0));
4223 Tmp2
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(1));
4224 // Perform the larger operation, then convert back
4225 Tmp1
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
, Tmp2
);
4226 Results
.push_back(DAG
.getNode(TruncOp
, dl
, OVT
, Tmp1
));
4229 case ISD::UMUL_LOHI
:
4230 case ISD::SMUL_LOHI
: {
4231 // Promote to a multiply in a wider integer type.
4232 unsigned ExtOp
= Node
->getOpcode() == ISD::UMUL_LOHI
? ISD::ZERO_EXTEND
4234 Tmp1
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(0));
4235 Tmp2
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(1));
4236 Tmp1
= DAG
.getNode(ISD::MUL
, dl
, NVT
, Tmp1
, Tmp2
);
4238 auto &DL
= DAG
.getDataLayout();
4239 unsigned OriginalSize
= OVT
.getScalarSizeInBits();
4241 ISD::SRL
, dl
, NVT
, Tmp1
,
4242 DAG
.getConstant(OriginalSize
, dl
, TLI
.getScalarShiftAmountTy(DL
, NVT
)));
4243 Results
.push_back(DAG
.getNode(ISD::TRUNCATE
, dl
, OVT
, Tmp1
));
4244 Results
.push_back(DAG
.getNode(ISD::TRUNCATE
, dl
, OVT
, Tmp2
));
4248 unsigned ExtOp
, TruncOp
;
4249 if (Node
->getValueType(0).isVector() ||
4250 Node
->getValueType(0).getSizeInBits() == NVT
.getSizeInBits()) {
4251 ExtOp
= ISD::BITCAST
;
4252 TruncOp
= ISD::BITCAST
;
4253 } else if (Node
->getValueType(0).isInteger()) {
4254 ExtOp
= ISD::ANY_EXTEND
;
4255 TruncOp
= ISD::TRUNCATE
;
4257 ExtOp
= ISD::FP_EXTEND
;
4258 TruncOp
= ISD::FP_ROUND
;
4260 Tmp1
= Node
->getOperand(0);
4261 // Promote each of the values to the new type.
4262 Tmp2
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(1));
4263 Tmp3
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(2));
4264 // Perform the larger operation, then round down.
4265 Tmp1
= DAG
.getSelect(dl
, NVT
, Tmp1
, Tmp2
, Tmp3
);
4266 Tmp1
->setFlags(Node
->getFlags());
4267 if (TruncOp
!= ISD::FP_ROUND
)
4268 Tmp1
= DAG
.getNode(TruncOp
, dl
, Node
->getValueType(0), Tmp1
);
4270 Tmp1
= DAG
.getNode(TruncOp
, dl
, Node
->getValueType(0), Tmp1
,
4271 DAG
.getIntPtrConstant(0, dl
));
4272 Results
.push_back(Tmp1
);
4275 case ISD::VECTOR_SHUFFLE
: {
4276 ArrayRef
<int> Mask
= cast
<ShuffleVectorSDNode
>(Node
)->getMask();
4278 // Cast the two input vectors.
4279 Tmp1
= DAG
.getNode(ISD::BITCAST
, dl
, NVT
, Node
->getOperand(0));
4280 Tmp2
= DAG
.getNode(ISD::BITCAST
, dl
, NVT
, Node
->getOperand(1));
4282 // Convert the shuffle mask to the right # elements.
4283 Tmp1
= ShuffleWithNarrowerEltType(NVT
, OVT
, dl
, Tmp1
, Tmp2
, Mask
);
4284 Tmp1
= DAG
.getNode(ISD::BITCAST
, dl
, OVT
, Tmp1
);
4285 Results
.push_back(Tmp1
);
4289 unsigned ExtOp
= ISD::FP_EXTEND
;
4290 if (NVT
.isInteger()) {
4291 ISD::CondCode CCCode
=
4292 cast
<CondCodeSDNode
>(Node
->getOperand(2))->get();
4293 ExtOp
= isSignedIntSetCC(CCCode
) ? ISD::SIGN_EXTEND
: ISD::ZERO_EXTEND
;
4295 Tmp1
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(0));
4296 Tmp2
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(1));
4297 Results
.push_back(DAG
.getNode(ISD::SETCC
, dl
, Node
->getValueType(0), Tmp1
,
4298 Tmp2
, Node
->getOperand(2), Node
->getFlags()));
4302 unsigned ExtOp
= ISD::FP_EXTEND
;
4303 if (NVT
.isInteger()) {
4304 ISD::CondCode CCCode
=
4305 cast
<CondCodeSDNode
>(Node
->getOperand(1))->get();
4306 ExtOp
= isSignedIntSetCC(CCCode
) ? ISD::SIGN_EXTEND
: ISD::ZERO_EXTEND
;
4308 Tmp1
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(2));
4309 Tmp2
= DAG
.getNode(ExtOp
, dl
, NVT
, Node
->getOperand(3));
4310 Results
.push_back(DAG
.getNode(ISD::BR_CC
, dl
, Node
->getValueType(0),
4311 Node
->getOperand(0), Node
->getOperand(1),
4312 Tmp1
, Tmp2
, Node
->getOperand(4)));
4323 Tmp1
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4324 Tmp2
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(1));
4325 Tmp3
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
, Tmp2
,
4327 Results
.push_back(DAG
.getNode(ISD::FP_ROUND
, dl
, OVT
,
4328 Tmp3
, DAG
.getIntPtrConstant(0, dl
)));
4331 Tmp1
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4332 Tmp2
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(1));
4333 Tmp3
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(2));
4335 DAG
.getNode(ISD::FP_ROUND
, dl
, OVT
,
4336 DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
, Tmp2
, Tmp3
),
4337 DAG
.getIntPtrConstant(0, dl
)));
4339 case ISD::FCOPYSIGN
:
4341 Tmp1
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4342 Tmp2
= Node
->getOperand(1);
4343 Tmp3
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
, Tmp2
);
4345 // fcopysign doesn't change anything but the sign bit, so
4346 // (fp_round (fcopysign (fpext a), b))
4348 // (fp_round (fpext a))
4349 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4350 const bool isTrunc
= (Node
->getOpcode() == ISD::FCOPYSIGN
);
4351 Results
.push_back(DAG
.getNode(ISD::FP_ROUND
, dl
, OVT
,
4352 Tmp3
, DAG
.getIntPtrConstant(isTrunc
, dl
)));
4358 case ISD::FNEARBYINT
:
4371 Tmp1
= DAG
.getNode(ISD::FP_EXTEND
, dl
, NVT
, Node
->getOperand(0));
4372 Tmp2
= DAG
.getNode(Node
->getOpcode(), dl
, NVT
, Tmp1
);
4373 Results
.push_back(DAG
.getNode(ISD::FP_ROUND
, dl
, OVT
,
4374 Tmp2
, DAG
.getIntPtrConstant(0, dl
)));
4376 case ISD::BUILD_VECTOR
: {
4377 MVT EltVT
= OVT
.getVectorElementType();
4378 MVT NewEltVT
= NVT
.getVectorElementType();
4380 // Handle bitcasts to a different vector type with the same total bit size
4382 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4384 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4386 assert(NVT
.isVector() && OVT
.getSizeInBits() == NVT
.getSizeInBits() &&
4387 "Invalid promote type for build_vector");
4388 assert(NewEltVT
.bitsLT(EltVT
) && "not handled");
4390 MVT MidVT
= getPromotedVectorElementType(TLI
, EltVT
, NewEltVT
);
4392 SmallVector
<SDValue
, 8> NewOps
;
4393 for (unsigned I
= 0, E
= Node
->getNumOperands(); I
!= E
; ++I
) {
4394 SDValue Op
= Node
->getOperand(I
);
4395 NewOps
.push_back(DAG
.getNode(ISD::BITCAST
, SDLoc(Op
), MidVT
, Op
));
4399 SDValue Concat
= DAG
.getNode(ISD::CONCAT_VECTORS
, SL
, NVT
, NewOps
);
4400 SDValue CvtVec
= DAG
.getNode(ISD::BITCAST
, SL
, OVT
, Concat
);
4401 Results
.push_back(CvtVec
);
4404 case ISD::EXTRACT_VECTOR_ELT
: {
4405 MVT EltVT
= OVT
.getVectorElementType();
4406 MVT NewEltVT
= NVT
.getVectorElementType();
4408 // Handle bitcasts to a different vector type with the same total bit size.
4410 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4412 // v4i32:castx = bitcast x:v2i64
4415 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4416 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4419 assert(NVT
.isVector() && OVT
.getSizeInBits() == NVT
.getSizeInBits() &&
4420 "Invalid promote type for extract_vector_elt");
4421 assert(NewEltVT
.bitsLT(EltVT
) && "not handled");
4423 MVT MidVT
= getPromotedVectorElementType(TLI
, EltVT
, NewEltVT
);
4424 unsigned NewEltsPerOldElt
= MidVT
.getVectorNumElements();
4426 SDValue Idx
= Node
->getOperand(1);
4427 EVT IdxVT
= Idx
.getValueType();
4429 SDValue Factor
= DAG
.getConstant(NewEltsPerOldElt
, SL
, IdxVT
);
4430 SDValue NewBaseIdx
= DAG
.getNode(ISD::MUL
, SL
, IdxVT
, Idx
, Factor
);
4432 SDValue CastVec
= DAG
.getNode(ISD::BITCAST
, SL
, NVT
, Node
->getOperand(0));
4434 SmallVector
<SDValue
, 8> NewOps
;
4435 for (unsigned I
= 0; I
< NewEltsPerOldElt
; ++I
) {
4436 SDValue IdxOffset
= DAG
.getConstant(I
, SL
, IdxVT
);
4437 SDValue TmpIdx
= DAG
.getNode(ISD::ADD
, SL
, IdxVT
, NewBaseIdx
, IdxOffset
);
4439 SDValue Elt
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, SL
, NewEltVT
,
4441 NewOps
.push_back(Elt
);
4444 SDValue NewVec
= DAG
.getBuildVector(MidVT
, SL
, NewOps
);
4445 Results
.push_back(DAG
.getNode(ISD::BITCAST
, SL
, EltVT
, NewVec
));
4448 case ISD::INSERT_VECTOR_ELT
: {
4449 MVT EltVT
= OVT
.getVectorElementType();
4450 MVT NewEltVT
= NVT
.getVectorElementType();
4452 // Handle bitcasts to a different vector type with the same total bit size
4454 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4456 // v4i32:castx = bitcast x:v2i64
4457 // v2i32:casty = bitcast y:i64
4460 // (v4i32 insert_vector_elt
4461 // (v4i32 insert_vector_elt v4i32:castx,
4462 // (extract_vector_elt casty, 0), 2 * z),
4463 // (extract_vector_elt casty, 1), (2 * z + 1))
4465 assert(NVT
.isVector() && OVT
.getSizeInBits() == NVT
.getSizeInBits() &&
4466 "Invalid promote type for insert_vector_elt");
4467 assert(NewEltVT
.bitsLT(EltVT
) && "not handled");
4469 MVT MidVT
= getPromotedVectorElementType(TLI
, EltVT
, NewEltVT
);
4470 unsigned NewEltsPerOldElt
= MidVT
.getVectorNumElements();
4472 SDValue Val
= Node
->getOperand(1);
4473 SDValue Idx
= Node
->getOperand(2);
4474 EVT IdxVT
= Idx
.getValueType();
4477 SDValue Factor
= DAG
.getConstant(NewEltsPerOldElt
, SDLoc(), IdxVT
);
4478 SDValue NewBaseIdx
= DAG
.getNode(ISD::MUL
, SL
, IdxVT
, Idx
, Factor
);
4480 SDValue CastVec
= DAG
.getNode(ISD::BITCAST
, SL
, NVT
, Node
->getOperand(0));
4481 SDValue CastVal
= DAG
.getNode(ISD::BITCAST
, SL
, MidVT
, Val
);
4483 SDValue NewVec
= CastVec
;
4484 for (unsigned I
= 0; I
< NewEltsPerOldElt
; ++I
) {
4485 SDValue IdxOffset
= DAG
.getConstant(I
, SL
, IdxVT
);
4486 SDValue InEltIdx
= DAG
.getNode(ISD::ADD
, SL
, IdxVT
, NewBaseIdx
, IdxOffset
);
4488 SDValue Elt
= DAG
.getNode(ISD::EXTRACT_VECTOR_ELT
, SL
, NewEltVT
,
4489 CastVal
, IdxOffset
);
4491 NewVec
= DAG
.getNode(ISD::INSERT_VECTOR_ELT
, SL
, NVT
,
4492 NewVec
, Elt
, InEltIdx
);
4495 Results
.push_back(DAG
.getNode(ISD::BITCAST
, SL
, OVT
, NewVec
));
4498 case ISD::SCALAR_TO_VECTOR
: {
4499 MVT EltVT
= OVT
.getVectorElementType();
4500 MVT NewEltVT
= NVT
.getVectorElementType();
4502 // Handle bitcasts to different vector type with the same total bit size.
4504 // e.g. v2i64 = scalar_to_vector x:i64
4506 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4509 MVT MidVT
= getPromotedVectorElementType(TLI
, EltVT
, NewEltVT
);
4510 SDValue Val
= Node
->getOperand(0);
4513 SDValue CastVal
= DAG
.getNode(ISD::BITCAST
, SL
, MidVT
, Val
);
4514 SDValue Undef
= DAG
.getUNDEF(MidVT
);
4516 SmallVector
<SDValue
, 8> NewElts
;
4517 NewElts
.push_back(CastVal
);
4518 for (unsigned I
= 1, NElts
= OVT
.getVectorNumElements(); I
!= NElts
; ++I
)
4519 NewElts
.push_back(Undef
);
4521 SDValue Concat
= DAG
.getNode(ISD::CONCAT_VECTORS
, SL
, NVT
, NewElts
);
4522 SDValue CvtVec
= DAG
.getNode(ISD::BITCAST
, SL
, OVT
, Concat
);
4523 Results
.push_back(CvtVec
);
4526 case ISD::ATOMIC_SWAP
: {
4527 AtomicSDNode
*AM
= cast
<AtomicSDNode
>(Node
);
4529 SDValue CastVal
= DAG
.getNode(ISD::BITCAST
, SL
, NVT
, AM
->getVal());
4530 assert(NVT
.getSizeInBits() == OVT
.getSizeInBits() &&
4531 "unexpected promotion type");
4532 assert(AM
->getMemoryVT().getSizeInBits() == NVT
.getSizeInBits() &&
4533 "unexpected atomic_swap with illegal type");
4536 = DAG
.getAtomic(ISD::ATOMIC_SWAP
, SL
, NVT
,
4537 DAG
.getVTList(NVT
, MVT::Other
),
4538 { AM
->getChain(), AM
->getBasePtr(), CastVal
},
4539 AM
->getMemOperand());
4540 Results
.push_back(DAG
.getNode(ISD::BITCAST
, SL
, OVT
, NewAtomic
));
4541 Results
.push_back(NewAtomic
.getValue(1));
4546 // Replace the original node with the legalized result.
4547 if (!Results
.empty()) {
4548 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4549 ReplaceNode(Node
, Results
.data());
4551 LLVM_DEBUG(dbgs() << "Could not promote node\n");
4554 /// This is the entry point for the file.
4555 void SelectionDAG::Legalize() {
4556 AssignTopologicalOrder();
4558 SmallPtrSet
<SDNode
*, 16> LegalizedNodes
;
4559 // Use a delete listener to remove nodes which were deleted during
4560 // legalization from LegalizeNodes. This is needed to handle the situation
4561 // where a new node is allocated by the object pool to the same address of a
4562 // previously deleted node.
4563 DAGNodeDeletedListener
DeleteListener(
4565 [&LegalizedNodes
](SDNode
*N
, SDNode
*E
) { LegalizedNodes
.erase(N
); });
4567 SelectionDAGLegalize
Legalizer(*this, LegalizedNodes
);
4569 // Visit all the nodes. We start in topological order, so that we see
4570 // nodes with their original operands intact. Legalization can produce
4571 // new nodes which may themselves need to be legalized. Iterate until all
4572 // nodes have been legalized.
4574 bool AnyLegalized
= false;
4575 for (auto NI
= allnodes_end(); NI
!= allnodes_begin();) {
4579 if (N
->use_empty() && N
!= getRoot().getNode()) {
4585 if (LegalizedNodes
.insert(N
).second
) {
4586 AnyLegalized
= true;
4587 Legalizer
.LegalizeOp(N
);
4589 if (N
->use_empty() && N
!= getRoot().getNode()) {
4600 // Remove dead nodes now.
4604 bool SelectionDAG::LegalizeOp(SDNode
*N
,
4605 SmallSetVector
<SDNode
*, 16> &UpdatedNodes
) {
4606 SmallPtrSet
<SDNode
*, 16> LegalizedNodes
;
4607 SelectionDAGLegalize
Legalizer(*this, LegalizedNodes
, &UpdatedNodes
);
4609 // Directly insert the node in question, and legalize it. This will recurse
4610 // as needed through operands.
4611 LegalizedNodes
.insert(N
);
4612 Legalizer
.LegalizeOp(N
);
4614 return LegalizedNodes
.count(N
);