[Alignment][NFC] Use Align with TargetLowering::setMinFunctionAlignment
[llvm-core.git] / lib / CodeGen / SelectionDAG / LegalizeDAG.cpp
blob1236b7492a0e7df253908a98780845f68fe1bbfd
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
11 //===----------------------------------------------------------------------===//
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/CodeGen/ISDOpcodes.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineJumpTableInfo.h"
23 #include "llvm/CodeGen/MachineMemOperand.h"
24 #include "llvm/CodeGen/RuntimeLibcalls.h"
25 #include "llvm/CodeGen/SelectionDAG.h"
26 #include "llvm/CodeGen/SelectionDAGNodes.h"
27 #include "llvm/CodeGen/TargetFrameLowering.h"
28 #include "llvm/CodeGen/TargetLowering.h"
29 #include "llvm/CodeGen/TargetSubtargetInfo.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/CallingConv.h"
32 #include "llvm/IR/Constants.h"
33 #include "llvm/IR/DataLayout.h"
34 #include "llvm/IR/DerivedTypes.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/Metadata.h"
37 #include "llvm/IR/Type.h"
38 #include "llvm/Support/Casting.h"
39 #include "llvm/Support/Compiler.h"
40 #include "llvm/Support/Debug.h"
41 #include "llvm/Support/ErrorHandling.h"
42 #include "llvm/Support/MachineValueType.h"
43 #include "llvm/Support/MathExtras.h"
44 #include "llvm/Support/raw_ostream.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include <algorithm>
48 #include <cassert>
49 #include <cstdint>
50 #include <tuple>
51 #include <utility>
53 using namespace llvm;
55 #define DEBUG_TYPE "legalizedag"
57 namespace {
59 /// Keeps track of state when getting the sign of a floating-point value as an
60 /// integer.
61 struct FloatSignAsInt {
62 EVT FloatVT;
63 SDValue Chain;
64 SDValue FloatPtr;
65 SDValue IntPtr;
66 MachinePointerInfo IntPointerInfo;
67 MachinePointerInfo FloatPointerInfo;
68 SDValue IntValue;
69 APInt SignMask;
70 uint8_t SignBit;
73 //===----------------------------------------------------------------------===//
74 /// This takes an arbitrary SelectionDAG as input and
75 /// hacks on it until the target machine can handle it. This involves
76 /// eliminating value sizes the machine cannot handle (promoting small sizes to
77 /// large sizes or splitting up large values into small values) as well as
78 /// eliminating operations the machine cannot handle.
79 ///
80 /// This code also does a small amount of optimization and recognition of idioms
81 /// as part of its processing. For example, if a target does not support a
82 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
83 /// will attempt merge setcc and brc instructions into brcc's.
84 class SelectionDAGLegalize {
85 const TargetMachine &TM;
86 const TargetLowering &TLI;
87 SelectionDAG &DAG;
89 /// The set of nodes which have already been legalized. We hold a
90 /// reference to it in order to update as necessary on node deletion.
91 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 /// A set of all the nodes updated during legalization.
94 SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 EVT getSetCCResultType(EVT VT) const {
97 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
100 // Libcall insertion helpers.
102 public:
103 SelectionDAGLegalize(SelectionDAG &DAG,
104 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
105 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
106 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
107 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 /// Legalizes the given operation.
110 void LegalizeOp(SDNode *Node);
112 private:
113 SDValue OptimizeFloatStore(StoreSDNode *ST);
115 void LegalizeLoadOps(SDNode *Node);
116 void LegalizeStoreOps(SDNode *Node);
118 /// Some targets cannot handle a variable
119 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
120 /// is necessary to spill the vector being inserted into to memory, perform
121 /// the insert there, and then read the result back.
122 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123 const SDLoc &dl);
124 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125 const SDLoc &dl);
127 /// Return a vector shuffle operation which
128 /// performs the same shuffe in terms of order or result bytes, but on a type
129 /// whose vector element type is narrower than the original shuffle type.
130 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
131 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
132 SDValue N1, SDValue N2,
133 ArrayRef<int> Mask) const;
135 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
136 bool &NeedInvert, const SDLoc &dl);
138 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
141 SDNode *Node, bool isSigned);
142 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
143 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
144 RTLIB::Libcall Call_F128,
145 RTLIB::Libcall Call_PPCF128);
146 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
147 RTLIB::Libcall Call_I8,
148 RTLIB::Libcall Call_I16,
149 RTLIB::Libcall Call_I32,
150 RTLIB::Libcall Call_I64,
151 RTLIB::Libcall Call_I128);
152 SDValue ExpandArgFPLibCall(SDNode *Node,
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128);
156 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
159 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160 const SDLoc &dl);
161 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162 const SDLoc &dl, SDValue ChainIn);
163 SDValue ExpandBUILD_VECTOR(SDNode *Node);
164 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
165 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
166 SmallVectorImpl<SDValue> &Results);
167 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
168 SDValue Value) const;
169 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
170 SDValue NewIntValue) const;
171 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
172 SDValue ExpandFABS(SDNode *Node) const;
173 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
174 const SDLoc &dl);
175 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
176 const SDLoc &dl);
177 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
178 const SDLoc &dl);
180 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
181 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
183 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
184 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
185 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
187 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
188 SDValue ExpandConstant(ConstantSDNode *CP);
190 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
191 bool ExpandNode(SDNode *Node);
192 void ConvertNodeToLibcall(SDNode *Node);
193 void PromoteNode(SDNode *Node);
195 public:
196 // Node replacement helpers
198 void ReplacedNode(SDNode *N) {
199 LegalizedNodes.erase(N);
200 if (UpdatedNodes)
201 UpdatedNodes->insert(N);
204 void ReplaceNode(SDNode *Old, SDNode *New) {
205 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
206 dbgs() << " with: "; New->dump(&DAG));
208 assert(Old->getNumValues() == New->getNumValues() &&
209 "Replacing one node with another that produces a different number "
210 "of values!");
211 DAG.ReplaceAllUsesWith(Old, New);
212 if (UpdatedNodes)
213 UpdatedNodes->insert(New);
214 ReplacedNode(Old);
217 void ReplaceNode(SDValue Old, SDValue New) {
218 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
219 dbgs() << " with: "; New->dump(&DAG));
221 DAG.ReplaceAllUsesWith(Old, New);
222 if (UpdatedNodes)
223 UpdatedNodes->insert(New.getNode());
224 ReplacedNode(Old.getNode());
227 void ReplaceNode(SDNode *Old, const SDValue *New) {
228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
230 DAG.ReplaceAllUsesWith(Old, New);
231 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
232 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
233 New[i]->dump(&DAG));
234 if (UpdatedNodes)
235 UpdatedNodes->insert(New[i].getNode());
237 ReplacedNode(Old);
240 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
242 dbgs() << " with: "; New->dump(&DAG));
244 DAG.ReplaceAllUsesOfValueWith(Old, New);
245 if (UpdatedNodes)
246 UpdatedNodes->insert(New.getNode());
247 ReplacedNode(Old.getNode());
251 } // end anonymous namespace
253 /// Return a vector shuffle operation which
254 /// performs the same shuffle in terms of order or result bytes, but on a type
255 /// whose vector element type is narrower than the original shuffle type.
256 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
257 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
258 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
259 ArrayRef<int> Mask) const {
260 unsigned NumMaskElts = VT.getVectorNumElements();
261 unsigned NumDestElts = NVT.getVectorNumElements();
262 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
264 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
266 if (NumEltsGrowth == 1)
267 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
269 SmallVector<int, 8> NewMask;
270 for (unsigned i = 0; i != NumMaskElts; ++i) {
271 int Idx = Mask[i];
272 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
273 if (Idx < 0)
274 NewMask.push_back(-1);
275 else
276 NewMask.push_back(Idx * NumEltsGrowth + j);
279 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
280 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
281 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
284 /// Expands the ConstantFP node to an integer constant or
285 /// a load from the constant pool.
286 SDValue
287 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
288 bool Extend = false;
289 SDLoc dl(CFP);
291 // If a FP immediate is precise when represented as a float and if the
292 // target can do an extending load from float to double, we put it into
293 // the constant pool as a float, even if it's is statically typed as a
294 // double. This shrinks FP constants and canonicalizes them for targets where
295 // an FP extending load is the same cost as a normal load (such as on the x87
296 // fp stack or PPC FP unit).
297 EVT VT = CFP->getValueType(0);
298 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
299 if (!UseCP) {
300 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
301 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
302 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
305 APFloat APF = CFP->getValueAPF();
306 EVT OrigVT = VT;
307 EVT SVT = VT;
309 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
310 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
311 if (!APF.isSignaling()) {
312 while (SVT != MVT::f32 && SVT != MVT::f16) {
313 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
314 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
315 // Only do this if the target has a native EXTLOAD instruction from
316 // smaller type.
317 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
318 TLI.ShouldShrinkFPConstant(OrigVT)) {
319 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
320 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
321 VT = SVT;
322 Extend = true;
327 SDValue CPIdx =
328 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
329 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
330 if (Extend) {
331 SDValue Result = DAG.getExtLoad(
332 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
333 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
334 Alignment);
335 return Result;
337 SDValue Result = DAG.getLoad(
338 OrigVT, dl, DAG.getEntryNode(), CPIdx,
339 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
340 return Result;
343 /// Expands the Constant node to a load from the constant pool.
344 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
345 SDLoc dl(CP);
346 EVT VT = CP->getValueType(0);
347 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
348 TLI.getPointerTy(DAG.getDataLayout()));
349 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
350 SDValue Result = DAG.getLoad(
351 VT, dl, DAG.getEntryNode(), CPIdx,
352 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
353 return Result;
356 /// Some target cannot handle a variable insertion index for the
357 /// INSERT_VECTOR_ELT instruction. In this case, it
358 /// is necessary to spill the vector being inserted into to memory, perform
359 /// the insert there, and then read the result back.
360 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
361 SDValue Val,
362 SDValue Idx,
363 const SDLoc &dl) {
364 SDValue Tmp1 = Vec;
365 SDValue Tmp2 = Val;
366 SDValue Tmp3 = Idx;
368 // If the target doesn't support this, we have to spill the input vector
369 // to a temporary stack slot, update the element, then reload it. This is
370 // badness. We could also load the value into a vector register (either
371 // with a "move to register" or "extload into register" instruction, then
372 // permute it into place, if the idx is a constant and if the idx is
373 // supported by the target.
374 EVT VT = Tmp1.getValueType();
375 EVT EltVT = VT.getVectorElementType();
376 SDValue StackPtr = DAG.CreateStackTemporary(VT);
378 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
380 // Store the vector.
381 SDValue Ch = DAG.getStore(
382 DAG.getEntryNode(), dl, Tmp1, StackPtr,
383 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
385 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
387 // Store the scalar value.
388 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
389 // Load the updated vector.
390 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
391 DAG.getMachineFunction(), SPFI));
394 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
395 SDValue Idx,
396 const SDLoc &dl) {
397 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
398 // SCALAR_TO_VECTOR requires that the type of the value being inserted
399 // match the element type of the vector being created, except for
400 // integers in which case the inserted value can be over width.
401 EVT EltVT = Vec.getValueType().getVectorElementType();
402 if (Val.getValueType() == EltVT ||
403 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
404 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
405 Vec.getValueType(), Val);
407 unsigned NumElts = Vec.getValueType().getVectorNumElements();
408 // We generate a shuffle of InVec and ScVec, so the shuffle mask
409 // should be 0,1,2,3,4,5... with the appropriate element replaced with
410 // elt 0 of the RHS.
411 SmallVector<int, 8> ShufOps;
412 for (unsigned i = 0; i != NumElts; ++i)
413 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
415 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
418 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
421 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
422 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
423 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
424 // FIXME: We shouldn't do this for TargetConstantFP's.
425 // FIXME: move this to the DAG Combiner! Note that we can't regress due
426 // to phase ordering between legalized code and the dag combiner. This
427 // probably means that we need to integrate dag combiner and legalizer
428 // together.
429 // We generally can't do this one for long doubles.
430 SDValue Chain = ST->getChain();
431 SDValue Ptr = ST->getBasePtr();
432 unsigned Alignment = ST->getAlignment();
433 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
434 AAMDNodes AAInfo = ST->getAAInfo();
435 SDLoc dl(ST);
436 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
437 if (CFP->getValueType(0) == MVT::f32 &&
438 TLI.isTypeLegal(MVT::i32)) {
439 SDValue Con = DAG.getConstant(CFP->getValueAPF().
440 bitcastToAPInt().zextOrTrunc(32),
441 SDLoc(CFP), MVT::i32);
442 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
443 MMOFlags, AAInfo);
446 if (CFP->getValueType(0) == MVT::f64) {
447 // If this target supports 64-bit registers, do a single 64-bit store.
448 if (TLI.isTypeLegal(MVT::i64)) {
449 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
450 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
451 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
452 Alignment, MMOFlags, AAInfo);
455 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
456 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
457 // stores. If the target supports neither 32- nor 64-bits, this
458 // xform is certainly not worth it.
459 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
460 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
461 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
462 if (DAG.getDataLayout().isBigEndian())
463 std::swap(Lo, Hi);
465 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
466 MMOFlags, AAInfo);
467 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
468 DAG.getConstant(4, dl, Ptr.getValueType()));
469 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
470 ST->getPointerInfo().getWithOffset(4),
471 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
473 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
477 return SDValue(nullptr, 0);
480 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
481 StoreSDNode *ST = cast<StoreSDNode>(Node);
482 SDValue Chain = ST->getChain();
483 SDValue Ptr = ST->getBasePtr();
484 SDLoc dl(Node);
486 unsigned Alignment = ST->getAlignment();
487 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
488 AAMDNodes AAInfo = ST->getAAInfo();
490 if (!ST->isTruncatingStore()) {
491 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
492 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
493 ReplaceNode(ST, OptStore);
494 return;
497 SDValue Value = ST->getValue();
498 MVT VT = Value.getSimpleValueType();
499 switch (TLI.getOperationAction(ISD::STORE, VT)) {
500 default: llvm_unreachable("This action is not supported yet!");
501 case TargetLowering::Legal: {
502 // If this is an unaligned store and the target doesn't support it,
503 // expand it.
504 EVT MemVT = ST->getMemoryVT();
505 const DataLayout &DL = DAG.getDataLayout();
506 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
507 *ST->getMemOperand())) {
508 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
509 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
510 ReplaceNode(SDValue(ST, 0), Result);
511 } else
512 LLVM_DEBUG(dbgs() << "Legal store\n");
513 break;
515 case TargetLowering::Custom: {
516 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
517 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
518 if (Res && Res != SDValue(Node, 0))
519 ReplaceNode(SDValue(Node, 0), Res);
520 return;
522 case TargetLowering::Promote: {
523 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
524 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
525 "Can only promote stores to same size type");
526 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
527 SDValue Result =
528 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
529 Alignment, MMOFlags, AAInfo);
530 ReplaceNode(SDValue(Node, 0), Result);
531 break;
534 return;
537 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
538 SDValue Value = ST->getValue();
539 EVT StVT = ST->getMemoryVT();
540 unsigned StWidth = StVT.getSizeInBits();
541 auto &DL = DAG.getDataLayout();
543 if (StWidth != StVT.getStoreSizeInBits()) {
544 // Promote to a byte-sized store with upper bits zero if not
545 // storing an integral number of bytes. For example, promote
546 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
547 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
548 StVT.getStoreSizeInBits());
549 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
550 SDValue Result =
551 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
552 Alignment, MMOFlags, AAInfo);
553 ReplaceNode(SDValue(Node, 0), Result);
554 } else if (StWidth & (StWidth - 1)) {
555 // If not storing a power-of-2 number of bits, expand as two stores.
556 assert(!StVT.isVector() && "Unsupported truncstore!");
557 unsigned LogStWidth = Log2_32(StWidth);
558 assert(LogStWidth < 32);
559 unsigned RoundWidth = 1 << LogStWidth;
560 assert(RoundWidth < StWidth);
561 unsigned ExtraWidth = StWidth - RoundWidth;
562 assert(ExtraWidth < RoundWidth);
563 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
564 "Store size not an integral number of bytes!");
565 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
566 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
567 SDValue Lo, Hi;
568 unsigned IncrementSize;
570 if (DL.isLittleEndian()) {
571 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
572 // Store the bottom RoundWidth bits.
573 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
574 RoundVT, Alignment, MMOFlags, AAInfo);
576 // Store the remaining ExtraWidth bits.
577 IncrementSize = RoundWidth / 8;
578 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
579 DAG.getConstant(IncrementSize, dl,
580 Ptr.getValueType()));
581 Hi = DAG.getNode(
582 ISD::SRL, dl, Value.getValueType(), Value,
583 DAG.getConstant(RoundWidth, dl,
584 TLI.getShiftAmountTy(Value.getValueType(), DL)));
585 Hi = DAG.getTruncStore(
586 Chain, dl, Hi, Ptr,
587 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
588 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
589 } else {
590 // Big endian - avoid unaligned stores.
591 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
592 // Store the top RoundWidth bits.
593 Hi = DAG.getNode(
594 ISD::SRL, dl, Value.getValueType(), Value,
595 DAG.getConstant(ExtraWidth, dl,
596 TLI.getShiftAmountTy(Value.getValueType(), DL)));
597 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
598 RoundVT, Alignment, MMOFlags, AAInfo);
600 // Store the remaining ExtraWidth bits.
601 IncrementSize = RoundWidth / 8;
602 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
603 DAG.getConstant(IncrementSize, dl,
604 Ptr.getValueType()));
605 Lo = DAG.getTruncStore(
606 Chain, dl, Value, Ptr,
607 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
608 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
611 // The order of the stores doesn't matter.
612 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
613 ReplaceNode(SDValue(Node, 0), Result);
614 } else {
615 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
616 default: llvm_unreachable("This action is not supported yet!");
617 case TargetLowering::Legal: {
618 EVT MemVT = ST->getMemoryVT();
619 // If this is an unaligned store and the target doesn't support it,
620 // expand it.
621 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
622 *ST->getMemOperand())) {
623 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
624 ReplaceNode(SDValue(ST, 0), Result);
626 break;
628 case TargetLowering::Custom: {
629 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
630 if (Res && Res != SDValue(Node, 0))
631 ReplaceNode(SDValue(Node, 0), Res);
632 return;
634 case TargetLowering::Expand:
635 assert(!StVT.isVector() &&
636 "Vector Stores are handled in LegalizeVectorOps");
638 SDValue Result;
640 // TRUNCSTORE:i16 i32 -> STORE i16
641 if (TLI.isTypeLegal(StVT)) {
642 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
643 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
644 Alignment, MMOFlags, AAInfo);
645 } else {
646 // The in-memory type isn't legal. Truncate to the type it would promote
647 // to, and then do a truncstore.
648 Value = DAG.getNode(ISD::TRUNCATE, dl,
649 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
650 Value);
651 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
652 StVT, Alignment, MMOFlags, AAInfo);
655 ReplaceNode(SDValue(Node, 0), Result);
656 break;
661 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
662 LoadSDNode *LD = cast<LoadSDNode>(Node);
663 SDValue Chain = LD->getChain(); // The chain.
664 SDValue Ptr = LD->getBasePtr(); // The base pointer.
665 SDValue Value; // The value returned by the load op.
666 SDLoc dl(Node);
668 ISD::LoadExtType ExtType = LD->getExtensionType();
669 if (ExtType == ISD::NON_EXTLOAD) {
670 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
671 MVT VT = Node->getSimpleValueType(0);
672 SDValue RVal = SDValue(Node, 0);
673 SDValue RChain = SDValue(Node, 1);
675 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
676 default: llvm_unreachable("This action is not supported yet!");
677 case TargetLowering::Legal: {
678 EVT MemVT = LD->getMemoryVT();
679 const DataLayout &DL = DAG.getDataLayout();
680 // If this is an unaligned load and the target doesn't support it,
681 // expand it.
682 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
683 *LD->getMemOperand())) {
684 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
686 break;
688 case TargetLowering::Custom:
689 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
690 RVal = Res;
691 RChain = Res.getValue(1);
693 break;
695 case TargetLowering::Promote: {
696 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
697 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
698 "Can only promote loads to same size type");
700 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
701 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
702 RChain = Res.getValue(1);
703 break;
706 if (RChain.getNode() != Node) {
707 assert(RVal.getNode() != Node && "Load must be completely replaced");
708 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
709 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
710 if (UpdatedNodes) {
711 UpdatedNodes->insert(RVal.getNode());
712 UpdatedNodes->insert(RChain.getNode());
714 ReplacedNode(Node);
716 return;
719 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
720 EVT SrcVT = LD->getMemoryVT();
721 unsigned SrcWidth = SrcVT.getSizeInBits();
722 unsigned Alignment = LD->getAlignment();
723 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
724 AAMDNodes AAInfo = LD->getAAInfo();
726 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
727 // Some targets pretend to have an i1 loading operation, and actually
728 // load an i8. This trick is correct for ZEXTLOAD because the top 7
729 // bits are guaranteed to be zero; it helps the optimizers understand
730 // that these bits are zero. It is also useful for EXTLOAD, since it
731 // tells the optimizers that those bits are undefined. It would be
732 // nice to have an effective generic way of getting these benefits...
733 // Until such a way is found, don't insist on promoting i1 here.
734 (SrcVT != MVT::i1 ||
735 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
736 TargetLowering::Promote)) {
737 // Promote to a byte-sized load if not loading an integral number of
738 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
739 unsigned NewWidth = SrcVT.getStoreSizeInBits();
740 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
741 SDValue Ch;
743 // The extra bits are guaranteed to be zero, since we stored them that
744 // way. A zext load from NVT thus automatically gives zext from SrcVT.
746 ISD::LoadExtType NewExtType =
747 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
749 SDValue Result =
750 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
751 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
753 Ch = Result.getValue(1); // The chain.
755 if (ExtType == ISD::SEXTLOAD)
756 // Having the top bits zero doesn't help when sign extending.
757 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
758 Result.getValueType(),
759 Result, DAG.getValueType(SrcVT));
760 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
761 // All the top bits are guaranteed to be zero - inform the optimizers.
762 Result = DAG.getNode(ISD::AssertZext, dl,
763 Result.getValueType(), Result,
764 DAG.getValueType(SrcVT));
766 Value = Result;
767 Chain = Ch;
768 } else if (SrcWidth & (SrcWidth - 1)) {
769 // If not loading a power-of-2 number of bits, expand as two loads.
770 assert(!SrcVT.isVector() && "Unsupported extload!");
771 unsigned LogSrcWidth = Log2_32(SrcWidth);
772 assert(LogSrcWidth < 32);
773 unsigned RoundWidth = 1 << LogSrcWidth;
774 assert(RoundWidth < SrcWidth);
775 unsigned ExtraWidth = SrcWidth - RoundWidth;
776 assert(ExtraWidth < RoundWidth);
777 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
778 "Load size not an integral number of bytes!");
779 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
780 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
781 SDValue Lo, Hi, Ch;
782 unsigned IncrementSize;
783 auto &DL = DAG.getDataLayout();
785 if (DL.isLittleEndian()) {
786 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
787 // Load the bottom RoundWidth bits.
788 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
789 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
790 AAInfo);
792 // Load the remaining ExtraWidth bits.
793 IncrementSize = RoundWidth / 8;
794 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
795 DAG.getConstant(IncrementSize, dl,
796 Ptr.getValueType()));
797 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
798 LD->getPointerInfo().getWithOffset(IncrementSize),
799 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
800 AAInfo);
802 // Build a factor node to remember that this load is independent of
803 // the other one.
804 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
805 Hi.getValue(1));
807 // Move the top bits to the right place.
808 Hi = DAG.getNode(
809 ISD::SHL, dl, Hi.getValueType(), Hi,
810 DAG.getConstant(RoundWidth, dl,
811 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
813 // Join the hi and lo parts.
814 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
815 } else {
816 // Big endian - avoid unaligned loads.
817 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
818 // Load the top RoundWidth bits.
819 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
820 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
821 AAInfo);
823 // Load the remaining ExtraWidth bits.
824 IncrementSize = RoundWidth / 8;
825 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
826 DAG.getConstant(IncrementSize, dl,
827 Ptr.getValueType()));
828 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
829 LD->getPointerInfo().getWithOffset(IncrementSize),
830 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
831 AAInfo);
833 // Build a factor node to remember that this load is independent of
834 // the other one.
835 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
836 Hi.getValue(1));
838 // Move the top bits to the right place.
839 Hi = DAG.getNode(
840 ISD::SHL, dl, Hi.getValueType(), Hi,
841 DAG.getConstant(ExtraWidth, dl,
842 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
844 // Join the hi and lo parts.
845 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
848 Chain = Ch;
849 } else {
850 bool isCustom = false;
851 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
852 SrcVT.getSimpleVT())) {
853 default: llvm_unreachable("This action is not supported yet!");
854 case TargetLowering::Custom:
855 isCustom = true;
856 LLVM_FALLTHROUGH;
857 case TargetLowering::Legal:
858 Value = SDValue(Node, 0);
859 Chain = SDValue(Node, 1);
861 if (isCustom) {
862 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
863 Value = Res;
864 Chain = Res.getValue(1);
866 } else {
867 // If this is an unaligned load and the target doesn't support it,
868 // expand it.
869 EVT MemVT = LD->getMemoryVT();
870 const DataLayout &DL = DAG.getDataLayout();
871 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
872 *LD->getMemOperand())) {
873 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
876 break;
878 case TargetLowering::Expand: {
879 EVT DestVT = Node->getValueType(0);
880 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
881 // If the source type is not legal, see if there is a legal extload to
882 // an intermediate type that we can then extend further.
883 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
884 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
885 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
886 // If we are loading a legal type, this is a non-extload followed by a
887 // full extend.
888 ISD::LoadExtType MidExtType =
889 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
891 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
892 SrcVT, LD->getMemOperand());
893 unsigned ExtendOp =
894 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
895 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
896 Chain = Load.getValue(1);
897 break;
900 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
901 // normal undefined upper bits behavior to allow using an in-reg extend
902 // with the illegal FP type, so load as an integer and do the
903 // from-integer conversion.
904 if (SrcVT.getScalarType() == MVT::f16) {
905 EVT ISrcVT = SrcVT.changeTypeToInteger();
906 EVT IDestVT = DestVT.changeTypeToInteger();
907 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
909 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
910 Ptr, ISrcVT, LD->getMemOperand());
911 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
912 Chain = Result.getValue(1);
913 break;
917 assert(!SrcVT.isVector() &&
918 "Vector Loads are handled in LegalizeVectorOps");
920 // FIXME: This does not work for vectors on most targets. Sign-
921 // and zero-extend operations are currently folded into extending
922 // loads, whether they are legal or not, and then we end up here
923 // without any support for legalizing them.
924 assert(ExtType != ISD::EXTLOAD &&
925 "EXTLOAD should always be supported!");
926 // Turn the unsupported load into an EXTLOAD followed by an
927 // explicit zero/sign extend inreg.
928 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
929 Node->getValueType(0),
930 Chain, Ptr, SrcVT,
931 LD->getMemOperand());
932 SDValue ValRes;
933 if (ExtType == ISD::SEXTLOAD)
934 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
935 Result.getValueType(),
936 Result, DAG.getValueType(SrcVT));
937 else
938 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
939 Value = ValRes;
940 Chain = Result.getValue(1);
941 break;
946 // Since loads produce two values, make sure to remember that we legalized
947 // both of them.
948 if (Chain.getNode() != Node) {
949 assert(Value.getNode() != Node && "Load must be completely replaced");
950 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
951 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
952 if (UpdatedNodes) {
953 UpdatedNodes->insert(Value.getNode());
954 UpdatedNodes->insert(Chain.getNode());
956 ReplacedNode(Node);
960 /// Return a legal replacement for the given operation, with all legal operands.
961 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
962 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
964 // Allow illegal target nodes and illegal registers.
965 if (Node->getOpcode() == ISD::TargetConstant ||
966 Node->getOpcode() == ISD::Register)
967 return;
969 #ifndef NDEBUG
970 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
971 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
972 TargetLowering::TypeLegal ||
973 TLI.isTypeLegal(Node->getValueType(i))) &&
974 "Unexpected illegal type!");
976 for (const SDValue &Op : Node->op_values())
977 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
978 TargetLowering::TypeLegal ||
979 TLI.isTypeLegal(Op.getValueType()) ||
980 Op.getOpcode() == ISD::TargetConstant ||
981 Op.getOpcode() == ISD::Register) &&
982 "Unexpected illegal type!");
983 #endif
985 // Figure out the correct action; the way to query this varies by opcode
986 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
987 bool SimpleFinishLegalizing = true;
988 switch (Node->getOpcode()) {
989 case ISD::INTRINSIC_W_CHAIN:
990 case ISD::INTRINSIC_WO_CHAIN:
991 case ISD::INTRINSIC_VOID:
992 case ISD::STACKSAVE:
993 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
994 break;
995 case ISD::GET_DYNAMIC_AREA_OFFSET:
996 Action = TLI.getOperationAction(Node->getOpcode(),
997 Node->getValueType(0));
998 break;
999 case ISD::VAARG:
1000 Action = TLI.getOperationAction(Node->getOpcode(),
1001 Node->getValueType(0));
1002 if (Action != TargetLowering::Promote)
1003 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1004 break;
1005 case ISD::FP_TO_FP16:
1006 case ISD::SINT_TO_FP:
1007 case ISD::UINT_TO_FP:
1008 case ISD::EXTRACT_VECTOR_ELT:
1009 case ISD::LROUND:
1010 case ISD::LLROUND:
1011 case ISD::LRINT:
1012 case ISD::LLRINT:
1013 Action = TLI.getOperationAction(Node->getOpcode(),
1014 Node->getOperand(0).getValueType());
1015 break;
1016 case ISD::FP_ROUND_INREG:
1017 case ISD::SIGN_EXTEND_INREG: {
1018 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1019 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1020 break;
1022 case ISD::ATOMIC_STORE:
1023 Action = TLI.getOperationAction(Node->getOpcode(),
1024 Node->getOperand(2).getValueType());
1025 break;
1026 case ISD::SELECT_CC:
1027 case ISD::SETCC:
1028 case ISD::BR_CC: {
1029 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1030 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1031 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1032 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1033 ISD::CondCode CCCode =
1034 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1035 Action = TLI.getCondCodeAction(CCCode, OpVT);
1036 if (Action == TargetLowering::Legal) {
1037 if (Node->getOpcode() == ISD::SELECT_CC)
1038 Action = TLI.getOperationAction(Node->getOpcode(),
1039 Node->getValueType(0));
1040 else
1041 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1043 break;
1045 case ISD::LOAD:
1046 case ISD::STORE:
1047 // FIXME: Model these properly. LOAD and STORE are complicated, and
1048 // STORE expects the unlegalized operand in some cases.
1049 SimpleFinishLegalizing = false;
1050 break;
1051 case ISD::CALLSEQ_START:
1052 case ISD::CALLSEQ_END:
1053 // FIXME: This shouldn't be necessary. These nodes have special properties
1054 // dealing with the recursive nature of legalization. Removing this
1055 // special case should be done as part of making LegalizeDAG non-recursive.
1056 SimpleFinishLegalizing = false;
1057 break;
1058 case ISD::EXTRACT_ELEMENT:
1059 case ISD::FLT_ROUNDS_:
1060 case ISD::MERGE_VALUES:
1061 case ISD::EH_RETURN:
1062 case ISD::FRAME_TO_ARGS_OFFSET:
1063 case ISD::EH_DWARF_CFA:
1064 case ISD::EH_SJLJ_SETJMP:
1065 case ISD::EH_SJLJ_LONGJMP:
1066 case ISD::EH_SJLJ_SETUP_DISPATCH:
1067 // These operations lie about being legal: when they claim to be legal,
1068 // they should actually be expanded.
1069 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1070 if (Action == TargetLowering::Legal)
1071 Action = TargetLowering::Expand;
1072 break;
1073 case ISD::INIT_TRAMPOLINE:
1074 case ISD::ADJUST_TRAMPOLINE:
1075 case ISD::FRAMEADDR:
1076 case ISD::RETURNADDR:
1077 case ISD::ADDROFRETURNADDR:
1078 case ISD::SPONENTRY:
1079 // These operations lie about being legal: when they claim to be legal,
1080 // they should actually be custom-lowered.
1081 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1082 if (Action == TargetLowering::Legal)
1083 Action = TargetLowering::Custom;
1084 break;
1085 case ISD::READCYCLECOUNTER:
1086 // READCYCLECOUNTER returns an i64, even if type legalization might have
1087 // expanded that to several smaller types.
1088 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1089 break;
1090 case ISD::READ_REGISTER:
1091 case ISD::WRITE_REGISTER:
1092 // Named register is legal in the DAG, but blocked by register name
1093 // selection if not implemented by target (to chose the correct register)
1094 // They'll be converted to Copy(To/From)Reg.
1095 Action = TargetLowering::Legal;
1096 break;
1097 case ISD::DEBUGTRAP:
1098 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1099 if (Action == TargetLowering::Expand) {
1100 // replace ISD::DEBUGTRAP with ISD::TRAP
1101 SDValue NewVal;
1102 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1103 Node->getOperand(0));
1104 ReplaceNode(Node, NewVal.getNode());
1105 LegalizeOp(NewVal.getNode());
1106 return;
1108 break;
1109 case ISD::SADDSAT:
1110 case ISD::UADDSAT:
1111 case ISD::SSUBSAT:
1112 case ISD::USUBSAT: {
1113 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1114 break;
1116 case ISD::SMULFIX:
1117 case ISD::SMULFIXSAT:
1118 case ISD::UMULFIX: {
1119 unsigned Scale = Node->getConstantOperandVal(2);
1120 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1121 Node->getValueType(0), Scale);
1122 break;
1124 case ISD::MSCATTER:
1125 Action = TLI.getOperationAction(Node->getOpcode(),
1126 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1127 break;
1128 case ISD::MSTORE:
1129 Action = TLI.getOperationAction(Node->getOpcode(),
1130 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1131 break;
1132 case ISD::VECREDUCE_FADD:
1133 case ISD::VECREDUCE_FMUL:
1134 case ISD::VECREDUCE_ADD:
1135 case ISD::VECREDUCE_MUL:
1136 case ISD::VECREDUCE_AND:
1137 case ISD::VECREDUCE_OR:
1138 case ISD::VECREDUCE_XOR:
1139 case ISD::VECREDUCE_SMAX:
1140 case ISD::VECREDUCE_SMIN:
1141 case ISD::VECREDUCE_UMAX:
1142 case ISD::VECREDUCE_UMIN:
1143 case ISD::VECREDUCE_FMAX:
1144 case ISD::VECREDUCE_FMIN:
1145 Action = TLI.getOperationAction(
1146 Node->getOpcode(), Node->getOperand(0).getValueType());
1147 break;
1148 default:
1149 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1150 Action = TargetLowering::Legal;
1151 } else {
1152 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1154 break;
1157 if (SimpleFinishLegalizing) {
1158 SDNode *NewNode = Node;
1159 switch (Node->getOpcode()) {
1160 default: break;
1161 case ISD::SHL:
1162 case ISD::SRL:
1163 case ISD::SRA:
1164 case ISD::ROTL:
1165 case ISD::ROTR: {
1166 // Legalizing shifts/rotates requires adjusting the shift amount
1167 // to the appropriate width.
1168 SDValue Op0 = Node->getOperand(0);
1169 SDValue Op1 = Node->getOperand(1);
1170 if (!Op1.getValueType().isVector()) {
1171 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1172 // The getShiftAmountOperand() may create a new operand node or
1173 // return the existing one. If new operand is created we need
1174 // to update the parent node.
1175 // Do not try to legalize SAO here! It will be automatically legalized
1176 // in the next round.
1177 if (SAO != Op1)
1178 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1181 break;
1182 case ISD::FSHL:
1183 case ISD::FSHR:
1184 case ISD::SRL_PARTS:
1185 case ISD::SRA_PARTS:
1186 case ISD::SHL_PARTS: {
1187 // Legalizing shifts/rotates requires adjusting the shift amount
1188 // to the appropriate width.
1189 SDValue Op0 = Node->getOperand(0);
1190 SDValue Op1 = Node->getOperand(1);
1191 SDValue Op2 = Node->getOperand(2);
1192 if (!Op2.getValueType().isVector()) {
1193 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1194 // The getShiftAmountOperand() may create a new operand node or
1195 // return the existing one. If new operand is created we need
1196 // to update the parent node.
1197 if (SAO != Op2)
1198 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1200 break;
1204 if (NewNode != Node) {
1205 ReplaceNode(Node, NewNode);
1206 Node = NewNode;
1208 switch (Action) {
1209 case TargetLowering::Legal:
1210 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1211 return;
1212 case TargetLowering::Custom:
1213 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1214 // FIXME: The handling for custom lowering with multiple results is
1215 // a complete mess.
1216 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1217 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1218 return;
1220 if (Node->getNumValues() == 1) {
1221 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1222 // We can just directly replace this node with the lowered value.
1223 ReplaceNode(SDValue(Node, 0), Res);
1224 return;
1227 SmallVector<SDValue, 8> ResultVals;
1228 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1229 ResultVals.push_back(Res.getValue(i));
1230 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1231 ReplaceNode(Node, ResultVals.data());
1232 return;
1234 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1235 LLVM_FALLTHROUGH;
1236 case TargetLowering::Expand:
1237 if (ExpandNode(Node))
1238 return;
1239 LLVM_FALLTHROUGH;
1240 case TargetLowering::LibCall:
1241 ConvertNodeToLibcall(Node);
1242 return;
1243 case TargetLowering::Promote:
1244 PromoteNode(Node);
1245 return;
1249 switch (Node->getOpcode()) {
1250 default:
1251 #ifndef NDEBUG
1252 dbgs() << "NODE: ";
1253 Node->dump( &DAG);
1254 dbgs() << "\n";
1255 #endif
1256 llvm_unreachable("Do not know how to legalize this operator!");
1258 case ISD::CALLSEQ_START:
1259 case ISD::CALLSEQ_END:
1260 break;
1261 case ISD::LOAD:
1262 return LegalizeLoadOps(Node);
1263 case ISD::STORE:
1264 return LegalizeStoreOps(Node);
1268 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1269 SDValue Vec = Op.getOperand(0);
1270 SDValue Idx = Op.getOperand(1);
1271 SDLoc dl(Op);
1273 // Before we generate a new store to a temporary stack slot, see if there is
1274 // already one that we can use. There often is because when we scalarize
1275 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1276 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1277 // the vector. If all are expanded here, we don't want one store per vector
1278 // element.
1280 // Caches for hasPredecessorHelper
1281 SmallPtrSet<const SDNode *, 32> Visited;
1282 SmallVector<const SDNode *, 16> Worklist;
1283 Visited.insert(Op.getNode());
1284 Worklist.push_back(Idx.getNode());
1285 SDValue StackPtr, Ch;
1286 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1287 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1288 SDNode *User = *UI;
1289 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1290 if (ST->isIndexed() || ST->isTruncatingStore() ||
1291 ST->getValue() != Vec)
1292 continue;
1294 // Make sure that nothing else could have stored into the destination of
1295 // this store.
1296 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1297 continue;
1299 // If the index is dependent on the store we will introduce a cycle when
1300 // creating the load (the load uses the index, and by replacing the chain
1301 // we will make the index dependent on the load). Also, the store might be
1302 // dependent on the extractelement and introduce a cycle when creating
1303 // the load.
1304 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1305 ST->hasPredecessor(Op.getNode()))
1306 continue;
1308 StackPtr = ST->getBasePtr();
1309 Ch = SDValue(ST, 0);
1310 break;
1314 EVT VecVT = Vec.getValueType();
1316 if (!Ch.getNode()) {
1317 // Store the value to a temporary stack slot, then LOAD the returned part.
1318 StackPtr = DAG.CreateStackTemporary(VecVT);
1319 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1320 MachinePointerInfo());
1323 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1325 SDValue NewLoad;
1327 if (Op.getValueType().isVector())
1328 NewLoad =
1329 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1330 else
1331 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1332 MachinePointerInfo(),
1333 VecVT.getVectorElementType());
1335 // Replace the chain going out of the store, by the one out of the load.
1336 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1338 // We introduced a cycle though, so update the loads operands, making sure
1339 // to use the original store's chain as an incoming chain.
1340 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1341 NewLoad->op_end());
1342 NewLoadOperands[0] = Ch;
1343 NewLoad =
1344 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1345 return NewLoad;
1348 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1349 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1351 SDValue Vec = Op.getOperand(0);
1352 SDValue Part = Op.getOperand(1);
1353 SDValue Idx = Op.getOperand(2);
1354 SDLoc dl(Op);
1356 // Store the value to a temporary stack slot, then LOAD the returned part.
1357 EVT VecVT = Vec.getValueType();
1358 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1359 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1360 MachinePointerInfo PtrInfo =
1361 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1363 // First store the whole vector.
1364 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1366 // Then store the inserted part.
1367 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1369 // Store the subvector.
1370 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1372 // Finally, load the updated vector.
1373 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1376 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1377 // We can't handle this case efficiently. Allocate a sufficiently
1378 // aligned object on the stack, store each element into it, then load
1379 // the result as a vector.
1380 // Create the stack frame object.
1381 EVT VT = Node->getValueType(0);
1382 EVT EltVT = VT.getVectorElementType();
1383 SDLoc dl(Node);
1384 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1385 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1386 MachinePointerInfo PtrInfo =
1387 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1389 // Emit a store of each element to the stack slot.
1390 SmallVector<SDValue, 8> Stores;
1391 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1392 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1393 // Store (in the right endianness) the elements to memory.
1394 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1395 // Ignore undef elements.
1396 if (Node->getOperand(i).isUndef()) continue;
1398 unsigned Offset = TypeByteSize*i;
1400 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1401 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1403 // If the destination vector element type is narrower than the source
1404 // element type, only store the bits necessary.
1405 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1406 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1407 Node->getOperand(i), Idx,
1408 PtrInfo.getWithOffset(Offset), EltVT));
1409 } else
1410 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1411 Idx, PtrInfo.getWithOffset(Offset)));
1414 SDValue StoreChain;
1415 if (!Stores.empty()) // Not all undef elements?
1416 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1417 else
1418 StoreChain = DAG.getEntryNode();
1420 // Result is a load from the stack slot.
1421 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1424 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1425 /// containing the sign bit if the target has no integer value capable of
1426 /// holding all bits of the floating-point value.
1427 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1428 const SDLoc &DL,
1429 SDValue Value) const {
1430 EVT FloatVT = Value.getValueType();
1431 unsigned NumBits = FloatVT.getSizeInBits();
1432 State.FloatVT = FloatVT;
1433 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1434 // Convert to an integer of the same size.
1435 if (TLI.isTypeLegal(IVT)) {
1436 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1437 State.SignMask = APInt::getSignMask(NumBits);
1438 State.SignBit = NumBits - 1;
1439 return;
1442 auto &DataLayout = DAG.getDataLayout();
1443 // Store the float to memory, then load the sign part out as an integer.
1444 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1445 // First create a temporary that is aligned for both the load and store.
1446 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1447 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1448 // Then store the float to it.
1449 State.FloatPtr = StackPtr;
1450 MachineFunction &MF = DAG.getMachineFunction();
1451 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1452 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1453 State.FloatPointerInfo);
1455 SDValue IntPtr;
1456 if (DataLayout.isBigEndian()) {
1457 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1458 // Load out a legal integer with the same sign bit as the float.
1459 IntPtr = StackPtr;
1460 State.IntPointerInfo = State.FloatPointerInfo;
1461 } else {
1462 // Advance the pointer so that the loaded byte will contain the sign bit.
1463 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1464 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1465 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1466 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1467 ByteOffset);
1470 State.IntPtr = IntPtr;
1471 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1472 State.IntPointerInfo, MVT::i8);
1473 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1474 State.SignBit = 7;
1477 /// Replace the integer value produced by getSignAsIntValue() with a new value
1478 /// and cast the result back to a floating-point type.
1479 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1480 const SDLoc &DL,
1481 SDValue NewIntValue) const {
1482 if (!State.Chain)
1483 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1485 // Override the part containing the sign bit in the value stored on the stack.
1486 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1487 State.IntPointerInfo, MVT::i8);
1488 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1489 State.FloatPointerInfo);
1492 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1493 SDLoc DL(Node);
1494 SDValue Mag = Node->getOperand(0);
1495 SDValue Sign = Node->getOperand(1);
1497 // Get sign bit into an integer value.
1498 FloatSignAsInt SignAsInt;
1499 getSignAsIntValue(SignAsInt, DL, Sign);
1501 EVT IntVT = SignAsInt.IntValue.getValueType();
1502 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1503 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1504 SignMask);
1506 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1507 EVT FloatVT = Mag.getValueType();
1508 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1509 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1510 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1511 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1512 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1513 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1514 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1517 // Transform Mag value to integer, and clear the sign bit.
1518 FloatSignAsInt MagAsInt;
1519 getSignAsIntValue(MagAsInt, DL, Mag);
1520 EVT MagVT = MagAsInt.IntValue.getValueType();
1521 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1522 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1523 ClearSignMask);
1525 // Get the signbit at the right position for MagAsInt.
1526 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1527 EVT ShiftVT = IntVT;
1528 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1529 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1530 ShiftVT = MagVT;
1532 if (ShiftAmount > 0) {
1533 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1534 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1535 } else if (ShiftAmount < 0) {
1536 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1537 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1539 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1540 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1543 // Store the part with the modified sign and convert back to float.
1544 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1545 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1548 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1549 SDLoc DL(Node);
1550 SDValue Value = Node->getOperand(0);
1552 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1553 EVT FloatVT = Value.getValueType();
1554 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1555 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1556 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1559 // Transform value to integer, clear the sign bit and transform back.
1560 FloatSignAsInt ValueAsInt;
1561 getSignAsIntValue(ValueAsInt, DL, Value);
1562 EVT IntVT = ValueAsInt.IntValue.getValueType();
1563 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1564 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1565 ClearSignMask);
1566 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1569 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1570 SmallVectorImpl<SDValue> &Results) {
1571 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1572 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1573 " not tell us which reg is the stack pointer!");
1574 SDLoc dl(Node);
1575 EVT VT = Node->getValueType(0);
1576 SDValue Tmp1 = SDValue(Node, 0);
1577 SDValue Tmp2 = SDValue(Node, 1);
1578 SDValue Tmp3 = Node->getOperand(2);
1579 SDValue Chain = Tmp1.getOperand(0);
1581 // Chain the dynamic stack allocation so that it doesn't modify the stack
1582 // pointer when other instructions are using the stack.
1583 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1585 SDValue Size = Tmp2.getOperand(1);
1586 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1587 Chain = SP.getValue(1);
1588 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1589 unsigned StackAlign =
1590 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1591 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1592 if (Align > StackAlign)
1593 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1594 DAG.getConstant(-(uint64_t)Align, dl, VT));
1595 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1597 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1598 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1600 Results.push_back(Tmp1);
1601 Results.push_back(Tmp2);
1604 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1605 /// target.
1607 /// If the SETCC has been legalized using AND / OR, then the legalized node
1608 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1609 /// will be set to false.
1611 /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1612 /// then the values of LHS and RHS will be swapped, CC will be set to the
1613 /// new condition, and NeedInvert will be set to false.
1615 /// If the SETCC has been legalized using the inverse condcode, then LHS and
1616 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1617 /// will be set to true. The caller must invert the result of the SETCC with
1618 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1619 /// of a true/false result.
1621 /// \returns true if the SetCC has been legalized, false if it hasn't.
1622 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1623 SDValue &RHS, SDValue &CC,
1624 bool &NeedInvert,
1625 const SDLoc &dl) {
1626 MVT OpVT = LHS.getSimpleValueType();
1627 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1628 NeedInvert = false;
1629 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1630 default: llvm_unreachable("Unknown condition code action!");
1631 case TargetLowering::Legal:
1632 // Nothing to do.
1633 break;
1634 case TargetLowering::Expand: {
1635 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1636 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1637 std::swap(LHS, RHS);
1638 CC = DAG.getCondCode(InvCC);
1639 return true;
1641 // Swapping operands didn't work. Try inverting the condition.
1642 bool NeedSwap = false;
1643 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1644 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1645 // If inverting the condition is not enough, try swapping operands
1646 // on top of it.
1647 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1648 NeedSwap = true;
1650 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1651 CC = DAG.getCondCode(InvCC);
1652 NeedInvert = true;
1653 if (NeedSwap)
1654 std::swap(LHS, RHS);
1655 return true;
1658 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1659 unsigned Opc = 0;
1660 switch (CCCode) {
1661 default: llvm_unreachable("Don't know how to expand this condition!");
1662 case ISD::SETO:
1663 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
1664 && "If SETO is expanded, SETOEQ must be legal!");
1665 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1666 case ISD::SETUO:
1667 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)
1668 && "If SETUO is expanded, SETUNE must be legal!");
1669 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1670 case ISD::SETOEQ:
1671 case ISD::SETOGT:
1672 case ISD::SETOGE:
1673 case ISD::SETOLT:
1674 case ISD::SETOLE:
1675 case ISD::SETONE:
1676 case ISD::SETUEQ:
1677 case ISD::SETUNE:
1678 case ISD::SETUGT:
1679 case ISD::SETUGE:
1680 case ISD::SETULT:
1681 case ISD::SETULE:
1682 // If we are floating point, assign and break, otherwise fall through.
1683 if (!OpVT.isInteger()) {
1684 // We can use the 4th bit to tell if we are the unordered
1685 // or ordered version of the opcode.
1686 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1687 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1688 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1689 break;
1691 // Fallthrough if we are unsigned integer.
1692 LLVM_FALLTHROUGH;
1693 case ISD::SETLE:
1694 case ISD::SETGT:
1695 case ISD::SETGE:
1696 case ISD::SETLT:
1697 case ISD::SETNE:
1698 case ISD::SETEQ:
1699 // If all combinations of inverting the condition and swapping operands
1700 // didn't work then we have no means to expand the condition.
1701 llvm_unreachable("Don't know how to expand this condition!");
1704 SDValue SetCC1, SetCC2;
1705 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1706 // If we aren't the ordered or unorder operation,
1707 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1708 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1709 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1710 } else {
1711 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1712 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1713 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1715 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1716 RHS = SDValue();
1717 CC = SDValue();
1718 return true;
1721 return false;
1724 /// Emit a store/load combination to the stack. This stores
1725 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1726 /// a load from the stack slot to DestVT, extending it if needed.
1727 /// The resultant code need not be legal.
1728 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1729 EVT DestVT, const SDLoc &dl) {
1730 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1733 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1734 EVT DestVT, const SDLoc &dl,
1735 SDValue Chain) {
1736 // Create the stack frame object.
1737 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1738 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1739 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1741 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1742 int SPFI = StackPtrFI->getIndex();
1743 MachinePointerInfo PtrInfo =
1744 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1746 unsigned SrcSize = SrcOp.getValueSizeInBits();
1747 unsigned SlotSize = SlotVT.getSizeInBits();
1748 unsigned DestSize = DestVT.getSizeInBits();
1749 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1750 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1752 // Emit a store to the stack slot. Use a truncstore if the input value is
1753 // later than DestVT.
1754 SDValue Store;
1756 if (SrcSize > SlotSize)
1757 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1758 SlotVT, SrcAlign);
1759 else {
1760 assert(SrcSize == SlotSize && "Invalid store");
1761 Store =
1762 DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1765 // Result is a load from the stack slot.
1766 if (SlotSize == DestSize)
1767 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1769 assert(SlotSize < DestSize && "Unknown extension!");
1770 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1771 DestAlign);
1774 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1775 SDLoc dl(Node);
1776 // Create a vector sized/aligned stack slot, store the value to element #0,
1777 // then load the whole vector back out.
1778 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1780 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1781 int SPFI = StackPtrFI->getIndex();
1783 SDValue Ch = DAG.getTruncStore(
1784 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1785 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1786 Node->getValueType(0).getVectorElementType());
1787 return DAG.getLoad(
1788 Node->getValueType(0), dl, Ch, StackPtr,
1789 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1792 static bool
1793 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1794 const TargetLowering &TLI, SDValue &Res) {
1795 unsigned NumElems = Node->getNumOperands();
1796 SDLoc dl(Node);
1797 EVT VT = Node->getValueType(0);
1799 // Try to group the scalars into pairs, shuffle the pairs together, then
1800 // shuffle the pairs of pairs together, etc. until the vector has
1801 // been built. This will work only if all of the necessary shuffle masks
1802 // are legal.
1804 // We do this in two phases; first to check the legality of the shuffles,
1805 // and next, assuming that all shuffles are legal, to create the new nodes.
1806 for (int Phase = 0; Phase < 2; ++Phase) {
1807 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1808 NewIntermedVals;
1809 for (unsigned i = 0; i < NumElems; ++i) {
1810 SDValue V = Node->getOperand(i);
1811 if (V.isUndef())
1812 continue;
1814 SDValue Vec;
1815 if (Phase)
1816 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1817 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1820 while (IntermedVals.size() > 2) {
1821 NewIntermedVals.clear();
1822 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1823 // This vector and the next vector are shuffled together (simply to
1824 // append the one to the other).
1825 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1827 SmallVector<int, 16> FinalIndices;
1828 FinalIndices.reserve(IntermedVals[i].second.size() +
1829 IntermedVals[i+1].second.size());
1831 int k = 0;
1832 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1833 ++j, ++k) {
1834 ShuffleVec[k] = j;
1835 FinalIndices.push_back(IntermedVals[i].second[j]);
1837 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1838 ++j, ++k) {
1839 ShuffleVec[k] = NumElems + j;
1840 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1843 SDValue Shuffle;
1844 if (Phase)
1845 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1846 IntermedVals[i+1].first,
1847 ShuffleVec);
1848 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1849 return false;
1850 NewIntermedVals.push_back(
1851 std::make_pair(Shuffle, std::move(FinalIndices)));
1854 // If we had an odd number of defined values, then append the last
1855 // element to the array of new vectors.
1856 if ((IntermedVals.size() & 1) != 0)
1857 NewIntermedVals.push_back(IntermedVals.back());
1859 IntermedVals.swap(NewIntermedVals);
1862 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1863 "Invalid number of intermediate vectors");
1864 SDValue Vec1 = IntermedVals[0].first;
1865 SDValue Vec2;
1866 if (IntermedVals.size() > 1)
1867 Vec2 = IntermedVals[1].first;
1868 else if (Phase)
1869 Vec2 = DAG.getUNDEF(VT);
1871 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1872 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1873 ShuffleVec[IntermedVals[0].second[i]] = i;
1874 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1875 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1877 if (Phase)
1878 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1879 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1880 return false;
1883 return true;
1886 /// Expand a BUILD_VECTOR node on targets that don't
1887 /// support the operation, but do support the resultant vector type.
1888 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1889 unsigned NumElems = Node->getNumOperands();
1890 SDValue Value1, Value2;
1891 SDLoc dl(Node);
1892 EVT VT = Node->getValueType(0);
1893 EVT OpVT = Node->getOperand(0).getValueType();
1894 EVT EltVT = VT.getVectorElementType();
1896 // If the only non-undef value is the low element, turn this into a
1897 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1898 bool isOnlyLowElement = true;
1899 bool MoreThanTwoValues = false;
1900 bool isConstant = true;
1901 for (unsigned i = 0; i < NumElems; ++i) {
1902 SDValue V = Node->getOperand(i);
1903 if (V.isUndef())
1904 continue;
1905 if (i > 0)
1906 isOnlyLowElement = false;
1907 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1908 isConstant = false;
1910 if (!Value1.getNode()) {
1911 Value1 = V;
1912 } else if (!Value2.getNode()) {
1913 if (V != Value1)
1914 Value2 = V;
1915 } else if (V != Value1 && V != Value2) {
1916 MoreThanTwoValues = true;
1920 if (!Value1.getNode())
1921 return DAG.getUNDEF(VT);
1923 if (isOnlyLowElement)
1924 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1926 // If all elements are constants, create a load from the constant pool.
1927 if (isConstant) {
1928 SmallVector<Constant*, 16> CV;
1929 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1930 if (ConstantFPSDNode *V =
1931 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1932 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1933 } else if (ConstantSDNode *V =
1934 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1935 if (OpVT==EltVT)
1936 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1937 else {
1938 // If OpVT and EltVT don't match, EltVT is not legal and the
1939 // element values have been promoted/truncated earlier. Undo this;
1940 // we don't want a v16i8 to become a v16i32 for example.
1941 const ConstantInt *CI = V->getConstantIntValue();
1942 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1943 CI->getZExtValue()));
1945 } else {
1946 assert(Node->getOperand(i).isUndef());
1947 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1948 CV.push_back(UndefValue::get(OpNTy));
1951 Constant *CP = ConstantVector::get(CV);
1952 SDValue CPIdx =
1953 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1954 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1955 return DAG.getLoad(
1956 VT, dl, DAG.getEntryNode(), CPIdx,
1957 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1958 Alignment);
1961 SmallSet<SDValue, 16> DefinedValues;
1962 for (unsigned i = 0; i < NumElems; ++i) {
1963 if (Node->getOperand(i).isUndef())
1964 continue;
1965 DefinedValues.insert(Node->getOperand(i));
1968 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1969 if (!MoreThanTwoValues) {
1970 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1971 for (unsigned i = 0; i < NumElems; ++i) {
1972 SDValue V = Node->getOperand(i);
1973 if (V.isUndef())
1974 continue;
1975 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1977 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1978 // Get the splatted value into the low element of a vector register.
1979 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1980 SDValue Vec2;
1981 if (Value2.getNode())
1982 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1983 else
1984 Vec2 = DAG.getUNDEF(VT);
1986 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1987 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1989 } else {
1990 SDValue Res;
1991 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1992 return Res;
1996 // Otherwise, we can't handle this case efficiently.
1997 return ExpandVectorBuildThroughStack(Node);
2000 // Expand a node into a call to a libcall. If the result value
2001 // does not fit into a register, return the lo part and set the hi part to the
2002 // by-reg argument. If it does fit into a single register, return the result
2003 // and leave the Hi part unset.
2004 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2005 bool isSigned) {
2006 TargetLowering::ArgListTy Args;
2007 TargetLowering::ArgListEntry Entry;
2008 for (const SDValue &Op : Node->op_values()) {
2009 EVT ArgVT = Op.getValueType();
2010 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2011 Entry.Node = Op;
2012 Entry.Ty = ArgTy;
2013 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2014 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2015 Args.push_back(Entry);
2017 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2018 TLI.getPointerTy(DAG.getDataLayout()));
2020 EVT RetVT = Node->getValueType(0);
2021 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2023 // By default, the input chain to this libcall is the entry node of the
2024 // function. If the libcall is going to be emitted as a tail call then
2025 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2026 // node which is being folded has a non-entry input chain.
2027 SDValue InChain = DAG.getEntryNode();
2029 // isTailCall may be true since the callee does not reference caller stack
2030 // frame. Check if it's in the right position and that the return types match.
2031 SDValue TCChain = InChain;
2032 const Function &F = DAG.getMachineFunction().getFunction();
2033 bool isTailCall =
2034 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2035 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2036 if (isTailCall)
2037 InChain = TCChain;
2039 TargetLowering::CallLoweringInfo CLI(DAG);
2040 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2041 CLI.setDebugLoc(SDLoc(Node))
2042 .setChain(InChain)
2043 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2044 std::move(Args))
2045 .setTailCall(isTailCall)
2046 .setSExtResult(signExtend)
2047 .setZExtResult(!signExtend)
2048 .setIsPostTypeLegalization(true);
2050 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2052 if (!CallInfo.second.getNode()) {
2053 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2054 // It's a tailcall, return the chain (which is the DAG root).
2055 return DAG.getRoot();
2058 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2059 return CallInfo.first;
2062 // Expand a node into a call to a libcall. Similar to
2063 // ExpandLibCall except that the first operand is the in-chain.
2064 std::pair<SDValue, SDValue>
2065 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2066 SDNode *Node,
2067 bool isSigned) {
2068 SDValue InChain = Node->getOperand(0);
2070 TargetLowering::ArgListTy Args;
2071 TargetLowering::ArgListEntry Entry;
2072 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2073 EVT ArgVT = Node->getOperand(i).getValueType();
2074 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2075 Entry.Node = Node->getOperand(i);
2076 Entry.Ty = ArgTy;
2077 Entry.IsSExt = isSigned;
2078 Entry.IsZExt = !isSigned;
2079 Args.push_back(Entry);
2081 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2082 TLI.getPointerTy(DAG.getDataLayout()));
2084 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2086 TargetLowering::CallLoweringInfo CLI(DAG);
2087 CLI.setDebugLoc(SDLoc(Node))
2088 .setChain(InChain)
2089 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2090 std::move(Args))
2091 .setSExtResult(isSigned)
2092 .setZExtResult(!isSigned);
2094 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2096 return CallInfo;
2099 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2100 RTLIB::Libcall Call_F32,
2101 RTLIB::Libcall Call_F64,
2102 RTLIB::Libcall Call_F80,
2103 RTLIB::Libcall Call_F128,
2104 RTLIB::Libcall Call_PPCF128) {
2105 if (Node->isStrictFPOpcode())
2106 Node = DAG.mutateStrictFPToFP(Node);
2108 RTLIB::Libcall LC;
2109 switch (Node->getSimpleValueType(0).SimpleTy) {
2110 default: llvm_unreachable("Unexpected request for libcall!");
2111 case MVT::f32: LC = Call_F32; break;
2112 case MVT::f64: LC = Call_F64; break;
2113 case MVT::f80: LC = Call_F80; break;
2114 case MVT::f128: LC = Call_F128; break;
2115 case MVT::ppcf128: LC = Call_PPCF128; break;
2117 return ExpandLibCall(LC, Node, false);
2120 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2121 RTLIB::Libcall Call_I8,
2122 RTLIB::Libcall Call_I16,
2123 RTLIB::Libcall Call_I32,
2124 RTLIB::Libcall Call_I64,
2125 RTLIB::Libcall Call_I128) {
2126 RTLIB::Libcall LC;
2127 switch (Node->getSimpleValueType(0).SimpleTy) {
2128 default: llvm_unreachable("Unexpected request for libcall!");
2129 case MVT::i8: LC = Call_I8; break;
2130 case MVT::i16: LC = Call_I16; break;
2131 case MVT::i32: LC = Call_I32; break;
2132 case MVT::i64: LC = Call_I64; break;
2133 case MVT::i128: LC = Call_I128; break;
2135 return ExpandLibCall(LC, Node, isSigned);
2138 /// Expand the node to a libcall based on first argument type (for instance
2139 /// lround and its variant).
2140 SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2141 RTLIB::Libcall Call_F32,
2142 RTLIB::Libcall Call_F64,
2143 RTLIB::Libcall Call_F80,
2144 RTLIB::Libcall Call_F128,
2145 RTLIB::Libcall Call_PPCF128) {
2146 RTLIB::Libcall LC;
2147 switch (Node->getOperand(0).getValueType().getSimpleVT().SimpleTy) {
2148 default: llvm_unreachable("Unexpected request for libcall!");
2149 case MVT::f32: LC = Call_F32; break;
2150 case MVT::f64: LC = Call_F64; break;
2151 case MVT::f80: LC = Call_F80; break;
2152 case MVT::f128: LC = Call_F128; break;
2153 case MVT::ppcf128: LC = Call_PPCF128; break;
2156 return ExpandLibCall(LC, Node, false);
2159 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2160 void
2161 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2162 SmallVectorImpl<SDValue> &Results) {
2163 unsigned Opcode = Node->getOpcode();
2164 bool isSigned = Opcode == ISD::SDIVREM;
2166 RTLIB::Libcall LC;
2167 switch (Node->getSimpleValueType(0).SimpleTy) {
2168 default: llvm_unreachable("Unexpected request for libcall!");
2169 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2170 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2171 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2172 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2173 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2176 // The input chain to this libcall is the entry node of the function.
2177 // Legalizing the call will automatically add the previous call to the
2178 // dependence.
2179 SDValue InChain = DAG.getEntryNode();
2181 EVT RetVT = Node->getValueType(0);
2182 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2184 TargetLowering::ArgListTy Args;
2185 TargetLowering::ArgListEntry Entry;
2186 for (const SDValue &Op : Node->op_values()) {
2187 EVT ArgVT = Op.getValueType();
2188 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2189 Entry.Node = Op;
2190 Entry.Ty = ArgTy;
2191 Entry.IsSExt = isSigned;
2192 Entry.IsZExt = !isSigned;
2193 Args.push_back(Entry);
2196 // Also pass the return address of the remainder.
2197 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2198 Entry.Node = FIPtr;
2199 Entry.Ty = RetTy->getPointerTo();
2200 Entry.IsSExt = isSigned;
2201 Entry.IsZExt = !isSigned;
2202 Args.push_back(Entry);
2204 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2205 TLI.getPointerTy(DAG.getDataLayout()));
2207 SDLoc dl(Node);
2208 TargetLowering::CallLoweringInfo CLI(DAG);
2209 CLI.setDebugLoc(dl)
2210 .setChain(InChain)
2211 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2212 std::move(Args))
2213 .setSExtResult(isSigned)
2214 .setZExtResult(!isSigned);
2216 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2218 // Remainder is loaded back from the stack frame.
2219 SDValue Rem =
2220 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2221 Results.push_back(CallInfo.first);
2222 Results.push_back(Rem);
2225 /// Return true if sincos libcall is available.
2226 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2227 RTLIB::Libcall LC;
2228 switch (Node->getSimpleValueType(0).SimpleTy) {
2229 default: llvm_unreachable("Unexpected request for libcall!");
2230 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2231 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2232 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2233 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2234 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2236 return TLI.getLibcallName(LC) != nullptr;
2239 /// Only issue sincos libcall if both sin and cos are needed.
2240 static bool useSinCos(SDNode *Node) {
2241 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2242 ? ISD::FCOS : ISD::FSIN;
2244 SDValue Op0 = Node->getOperand(0);
2245 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2246 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2247 SDNode *User = *UI;
2248 if (User == Node)
2249 continue;
2250 // The other user might have been turned into sincos already.
2251 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2252 return true;
2254 return false;
2257 /// Issue libcalls to sincos to compute sin / cos pairs.
2258 void
2259 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2260 SmallVectorImpl<SDValue> &Results) {
2261 RTLIB::Libcall LC;
2262 switch (Node->getSimpleValueType(0).SimpleTy) {
2263 default: llvm_unreachable("Unexpected request for libcall!");
2264 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2265 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2266 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2267 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2268 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2271 // The input chain to this libcall is the entry node of the function.
2272 // Legalizing the call will automatically add the previous call to the
2273 // dependence.
2274 SDValue InChain = DAG.getEntryNode();
2276 EVT RetVT = Node->getValueType(0);
2277 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2279 TargetLowering::ArgListTy Args;
2280 TargetLowering::ArgListEntry Entry;
2282 // Pass the argument.
2283 Entry.Node = Node->getOperand(0);
2284 Entry.Ty = RetTy;
2285 Entry.IsSExt = false;
2286 Entry.IsZExt = false;
2287 Args.push_back(Entry);
2289 // Pass the return address of sin.
2290 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2291 Entry.Node = SinPtr;
2292 Entry.Ty = RetTy->getPointerTo();
2293 Entry.IsSExt = false;
2294 Entry.IsZExt = false;
2295 Args.push_back(Entry);
2297 // Also pass the return address of the cos.
2298 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2299 Entry.Node = CosPtr;
2300 Entry.Ty = RetTy->getPointerTo();
2301 Entry.IsSExt = false;
2302 Entry.IsZExt = false;
2303 Args.push_back(Entry);
2305 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2306 TLI.getPointerTy(DAG.getDataLayout()));
2308 SDLoc dl(Node);
2309 TargetLowering::CallLoweringInfo CLI(DAG);
2310 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2311 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2312 std::move(Args));
2314 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2316 Results.push_back(
2317 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2318 Results.push_back(
2319 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2322 /// This function is responsible for legalizing a
2323 /// INT_TO_FP operation of the specified operand when the target requests that
2324 /// we expand it. At this point, we know that the result and operand types are
2325 /// legal for the target.
2326 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2327 EVT DestVT,
2328 const SDLoc &dl) {
2329 EVT SrcVT = Op0.getValueType();
2331 // TODO: Should any fast-math-flags be set for the created nodes?
2332 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2333 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2334 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2335 "expansion\n");
2337 // Get the stack frame index of a 8 byte buffer.
2338 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2340 // word offset constant for Hi/Lo address computation
2341 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2342 StackSlot.getValueType());
2343 // set up Hi and Lo (into buffer) address based on endian
2344 SDValue Hi = StackSlot;
2345 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2346 StackSlot, WordOff);
2347 if (DAG.getDataLayout().isLittleEndian())
2348 std::swap(Hi, Lo);
2350 // if signed map to unsigned space
2351 SDValue Op0Mapped;
2352 if (isSigned) {
2353 // constant used to invert sign bit (signed to unsigned mapping)
2354 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2355 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2356 } else {
2357 Op0Mapped = Op0;
2359 // store the lo of the constructed double - based on integer input
2360 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2361 MachinePointerInfo());
2362 // initial hi portion of constructed double
2363 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2364 // store the hi of the constructed double - biased exponent
2365 SDValue Store2 =
2366 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2367 // load the constructed double
2368 SDValue Load =
2369 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2370 // FP constant to bias correct the final result
2371 SDValue Bias = DAG.getConstantFP(isSigned ?
2372 BitsToDouble(0x4330000080000000ULL) :
2373 BitsToDouble(0x4330000000000000ULL),
2374 dl, MVT::f64);
2375 // subtract the bias
2376 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2377 // final result
2378 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2379 return Result;
2381 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2382 // Code below here assumes !isSigned without checking again.
2384 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2386 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2387 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2388 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2389 Four = DAG.getIntPtrConstant(4, dl);
2390 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2391 SignSet, Four, Zero);
2393 // If the sign bit of the integer is set, the large number will be treated
2394 // as a negative number. To counteract this, the dynamic code adds an
2395 // offset depending on the data type.
2396 uint64_t FF;
2397 switch (SrcVT.getSimpleVT().SimpleTy) {
2398 default: llvm_unreachable("Unsupported integer type!");
2399 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2400 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2401 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2402 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2404 if (DAG.getDataLayout().isLittleEndian())
2405 FF <<= 32;
2406 Constant *FudgeFactor = ConstantInt::get(
2407 Type::getInt64Ty(*DAG.getContext()), FF);
2409 SDValue CPIdx =
2410 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2411 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2412 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2413 Alignment = std::min(Alignment, 4u);
2414 SDValue FudgeInReg;
2415 if (DestVT == MVT::f32)
2416 FudgeInReg = DAG.getLoad(
2417 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2418 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2419 Alignment);
2420 else {
2421 SDValue Load = DAG.getExtLoad(
2422 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2423 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2424 Alignment);
2425 HandleSDNode Handle(Load);
2426 LegalizeOp(Load.getNode());
2427 FudgeInReg = Handle.getValue();
2430 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2433 /// This function is responsible for legalizing a
2434 /// *INT_TO_FP operation of the specified operand when the target requests that
2435 /// we promote it. At this point, we know that the result and operand types are
2436 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2437 /// operation that takes a larger input.
2438 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2439 bool isSigned,
2440 const SDLoc &dl) {
2441 // First step, figure out the appropriate *INT_TO_FP operation to use.
2442 EVT NewInTy = LegalOp.getValueType();
2444 unsigned OpToUse = 0;
2446 // Scan for the appropriate larger type to use.
2447 while (true) {
2448 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2449 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2451 // If the target supports SINT_TO_FP of this type, use it.
2452 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2453 OpToUse = ISD::SINT_TO_FP;
2454 break;
2456 if (isSigned) continue;
2458 // If the target supports UINT_TO_FP of this type, use it.
2459 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2460 OpToUse = ISD::UINT_TO_FP;
2461 break;
2464 // Otherwise, try a larger type.
2467 // Okay, we found the operation and type to use. Zero extend our input to the
2468 // desired type then run the operation on it.
2469 return DAG.getNode(OpToUse, dl, DestVT,
2470 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2471 dl, NewInTy, LegalOp));
2474 /// This function is responsible for legalizing a
2475 /// FP_TO_*INT operation of the specified operand when the target requests that
2476 /// we promote it. At this point, we know that the result and operand types are
2477 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2478 /// operation that returns a larger result.
2479 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2480 bool isSigned,
2481 const SDLoc &dl) {
2482 // First step, figure out the appropriate FP_TO*INT operation to use.
2483 EVT NewOutTy = DestVT;
2485 unsigned OpToUse = 0;
2487 // Scan for the appropriate larger type to use.
2488 while (true) {
2489 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2490 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2492 // A larger signed type can hold all unsigned values of the requested type,
2493 // so using FP_TO_SINT is valid
2494 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2495 OpToUse = ISD::FP_TO_SINT;
2496 break;
2499 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2500 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2501 OpToUse = ISD::FP_TO_UINT;
2502 break;
2505 // Otherwise, try a larger type.
2508 // Okay, we found the operation and type to use.
2509 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2511 // Truncate the result of the extended FP_TO_*INT operation to the desired
2512 // size.
2513 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2516 /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2517 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2518 EVT VT = Op.getValueType();
2519 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2520 unsigned Sz = VT.getScalarSizeInBits();
2522 SDValue Tmp, Tmp2, Tmp3;
2524 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2525 // and finally the i1 pairs.
2526 // TODO: We can easily support i4/i2 legal types if any target ever does.
2527 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2528 // Create the masks - repeating the pattern every byte.
2529 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0));
2530 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC));
2531 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA));
2532 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F));
2533 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33));
2534 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55));
2536 // BSWAP if the type is wider than a single byte.
2537 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2539 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2540 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2541 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2542 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
2543 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
2544 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2546 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2547 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2548 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2549 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
2550 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
2551 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2553 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2554 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2555 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2556 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
2557 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
2558 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2559 return Tmp;
2562 Tmp = DAG.getConstant(0, dl, VT);
2563 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2564 if (I < J)
2565 Tmp2 =
2566 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2567 else
2568 Tmp2 =
2569 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2571 APInt Shift(Sz, 1);
2572 Shift <<= J;
2573 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2574 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2577 return Tmp;
2580 /// Open code the operations for BSWAP of the specified operation.
2581 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2582 EVT VT = Op.getValueType();
2583 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2584 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2585 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2586 default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2587 case MVT::i16:
2588 // Use a rotate by 8. This can be further expanded if necessary.
2589 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2590 case MVT::i32:
2591 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2592 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2593 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2594 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2595 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2596 DAG.getConstant(0xFF0000, dl, VT));
2597 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2598 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2599 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2600 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2601 case MVT::i64:
2602 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2603 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2604 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2605 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2606 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2607 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2608 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2609 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2610 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2611 DAG.getConstant(255ULL<<48, dl, VT));
2612 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2613 DAG.getConstant(255ULL<<40, dl, VT));
2614 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2615 DAG.getConstant(255ULL<<32, dl, VT));
2616 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2617 DAG.getConstant(255ULL<<24, dl, VT));
2618 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2619 DAG.getConstant(255ULL<<16, dl, VT));
2620 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2621 DAG.getConstant(255ULL<<8 , dl, VT));
2622 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2623 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2624 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2625 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2626 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2627 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2628 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2632 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2633 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2634 SmallVector<SDValue, 8> Results;
2635 SDLoc dl(Node);
2636 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2637 bool NeedInvert;
2638 switch (Node->getOpcode()) {
2639 case ISD::ABS:
2640 if (TLI.expandABS(Node, Tmp1, DAG))
2641 Results.push_back(Tmp1);
2642 break;
2643 case ISD::CTPOP:
2644 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2645 Results.push_back(Tmp1);
2646 break;
2647 case ISD::CTLZ:
2648 case ISD::CTLZ_ZERO_UNDEF:
2649 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2650 Results.push_back(Tmp1);
2651 break;
2652 case ISD::CTTZ:
2653 case ISD::CTTZ_ZERO_UNDEF:
2654 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2655 Results.push_back(Tmp1);
2656 break;
2657 case ISD::BITREVERSE:
2658 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2659 break;
2660 case ISD::BSWAP:
2661 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2662 break;
2663 case ISD::FRAMEADDR:
2664 case ISD::RETURNADDR:
2665 case ISD::FRAME_TO_ARGS_OFFSET:
2666 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2667 break;
2668 case ISD::EH_DWARF_CFA: {
2669 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2670 TLI.getPointerTy(DAG.getDataLayout()));
2671 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2672 CfaArg.getValueType(),
2673 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2674 CfaArg.getValueType()),
2675 CfaArg);
2676 SDValue FA = DAG.getNode(
2677 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2678 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2679 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2680 FA, Offset));
2681 break;
2683 case ISD::FLT_ROUNDS_:
2684 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2685 break;
2686 case ISD::EH_RETURN:
2687 case ISD::EH_LABEL:
2688 case ISD::PREFETCH:
2689 case ISD::VAEND:
2690 case ISD::EH_SJLJ_LONGJMP:
2691 // If the target didn't expand these, there's nothing to do, so just
2692 // preserve the chain and be done.
2693 Results.push_back(Node->getOperand(0));
2694 break;
2695 case ISD::READCYCLECOUNTER:
2696 // If the target didn't expand this, just return 'zero' and preserve the
2697 // chain.
2698 Results.append(Node->getNumValues() - 1,
2699 DAG.getConstant(0, dl, Node->getValueType(0)));
2700 Results.push_back(Node->getOperand(0));
2701 break;
2702 case ISD::EH_SJLJ_SETJMP:
2703 // If the target didn't expand this, just return 'zero' and preserve the
2704 // chain.
2705 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2706 Results.push_back(Node->getOperand(0));
2707 break;
2708 case ISD::ATOMIC_LOAD: {
2709 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2710 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2711 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2712 SDValue Swap = DAG.getAtomicCmpSwap(
2713 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2714 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2715 cast<AtomicSDNode>(Node)->getMemOperand());
2716 Results.push_back(Swap.getValue(0));
2717 Results.push_back(Swap.getValue(1));
2718 break;
2720 case ISD::ATOMIC_STORE: {
2721 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2722 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2723 cast<AtomicSDNode>(Node)->getMemoryVT(),
2724 Node->getOperand(0),
2725 Node->getOperand(1), Node->getOperand(2),
2726 cast<AtomicSDNode>(Node)->getMemOperand());
2727 Results.push_back(Swap.getValue(1));
2728 break;
2730 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2731 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2732 // splits out the success value as a comparison. Expanding the resulting
2733 // ATOMIC_CMP_SWAP will produce a libcall.
2734 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2735 SDValue Res = DAG.getAtomicCmpSwap(
2736 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2737 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2738 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2740 SDValue ExtRes = Res;
2741 SDValue LHS = Res;
2742 SDValue RHS = Node->getOperand(1);
2744 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2745 EVT OuterType = Node->getValueType(0);
2746 switch (TLI.getExtendForAtomicOps()) {
2747 case ISD::SIGN_EXTEND:
2748 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2749 DAG.getValueType(AtomicType));
2750 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2751 Node->getOperand(2), DAG.getValueType(AtomicType));
2752 ExtRes = LHS;
2753 break;
2754 case ISD::ZERO_EXTEND:
2755 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2756 DAG.getValueType(AtomicType));
2757 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2758 ExtRes = LHS;
2759 break;
2760 case ISD::ANY_EXTEND:
2761 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2762 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2763 break;
2764 default:
2765 llvm_unreachable("Invalid atomic op extension");
2768 SDValue Success =
2769 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2771 Results.push_back(ExtRes.getValue(0));
2772 Results.push_back(Success);
2773 Results.push_back(Res.getValue(1));
2774 break;
2776 case ISD::DYNAMIC_STACKALLOC:
2777 ExpandDYNAMIC_STACKALLOC(Node, Results);
2778 break;
2779 case ISD::MERGE_VALUES:
2780 for (unsigned i = 0; i < Node->getNumValues(); i++)
2781 Results.push_back(Node->getOperand(i));
2782 break;
2783 case ISD::UNDEF: {
2784 EVT VT = Node->getValueType(0);
2785 if (VT.isInteger())
2786 Results.push_back(DAG.getConstant(0, dl, VT));
2787 else {
2788 assert(VT.isFloatingPoint() && "Unknown value type!");
2789 Results.push_back(DAG.getConstantFP(0, dl, VT));
2791 break;
2793 case ISD::STRICT_FP_ROUND:
2794 // This expansion does not honor the "strict" properties anyway,
2795 // so prefer falling back to the non-strict operation if legal.
2796 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2797 Node->getValueType(0))
2798 == TargetLowering::Legal)
2799 break;
2800 Tmp1 = EmitStackConvert(Node->getOperand(1),
2801 Node->getValueType(0),
2802 Node->getValueType(0), dl, Node->getOperand(0));
2803 ReplaceNode(Node, Tmp1.getNode());
2804 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2805 return true;
2806 case ISD::FP_ROUND:
2807 case ISD::BITCAST:
2808 Tmp1 = EmitStackConvert(Node->getOperand(0),
2809 Node->getValueType(0),
2810 Node->getValueType(0), dl);
2811 Results.push_back(Tmp1);
2812 break;
2813 case ISD::STRICT_FP_EXTEND:
2814 // This expansion does not honor the "strict" properties anyway,
2815 // so prefer falling back to the non-strict operation if legal.
2816 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2817 Node->getValueType(0))
2818 == TargetLowering::Legal)
2819 break;
2820 Tmp1 = EmitStackConvert(Node->getOperand(1),
2821 Node->getOperand(1).getValueType(),
2822 Node->getValueType(0), dl, Node->getOperand(0));
2823 ReplaceNode(Node, Tmp1.getNode());
2824 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2825 return true;
2826 case ISD::FP_EXTEND:
2827 Tmp1 = EmitStackConvert(Node->getOperand(0),
2828 Node->getOperand(0).getValueType(),
2829 Node->getValueType(0), dl);
2830 Results.push_back(Tmp1);
2831 break;
2832 case ISD::SIGN_EXTEND_INREG: {
2833 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2834 EVT VT = Node->getValueType(0);
2836 // An in-register sign-extend of a boolean is a negation:
2837 // 'true' (1) sign-extended is -1.
2838 // 'false' (0) sign-extended is 0.
2839 // However, we must mask the high bits of the source operand because the
2840 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2842 // TODO: Do this for vectors too?
2843 if (ExtraVT.getSizeInBits() == 1) {
2844 SDValue One = DAG.getConstant(1, dl, VT);
2845 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2846 SDValue Zero = DAG.getConstant(0, dl, VT);
2847 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2848 Results.push_back(Neg);
2849 break;
2852 // NOTE: we could fall back on load/store here too for targets without
2853 // SRA. However, it is doubtful that any exist.
2854 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2855 unsigned BitsDiff = VT.getScalarSizeInBits() -
2856 ExtraVT.getScalarSizeInBits();
2857 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2858 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2859 Node->getOperand(0), ShiftCst);
2860 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2861 Results.push_back(Tmp1);
2862 break;
2864 case ISD::FP_ROUND_INREG: {
2865 // The only way we can lower this is to turn it into a TRUNCSTORE,
2866 // EXTLOAD pair, targeting a temporary location (a stack slot).
2868 // NOTE: there is a choice here between constantly creating new stack
2869 // slots and always reusing the same one. We currently always create
2870 // new ones, as reuse may inhibit scheduling.
2871 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2872 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2873 Node->getValueType(0), dl);
2874 Results.push_back(Tmp1);
2875 break;
2877 case ISD::UINT_TO_FP:
2878 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2879 Results.push_back(Tmp1);
2880 break;
2882 LLVM_FALLTHROUGH;
2883 case ISD::SINT_TO_FP:
2884 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2885 Node->getOperand(0), Node->getValueType(0), dl);
2886 Results.push_back(Tmp1);
2887 break;
2888 case ISD::FP_TO_SINT:
2889 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2890 Results.push_back(Tmp1);
2891 break;
2892 case ISD::STRICT_FP_TO_SINT:
2893 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2894 ReplaceNode(Node, Tmp1.getNode());
2895 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2896 return true;
2898 break;
2899 case ISD::FP_TO_UINT:
2900 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2901 Results.push_back(Tmp1);
2902 break;
2903 case ISD::STRICT_FP_TO_UINT:
2904 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2905 // Relink the chain.
2906 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2907 // Replace the new UINT result.
2908 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2909 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2910 return true;
2912 break;
2913 case ISD::LROUND:
2914 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
2915 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
2916 RTLIB::LROUND_F128,
2917 RTLIB::LROUND_PPCF128));
2918 break;
2919 case ISD::LLROUND:
2920 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
2921 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
2922 RTLIB::LLROUND_F128,
2923 RTLIB::LLROUND_PPCF128));
2924 break;
2925 case ISD::LRINT:
2926 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
2927 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
2928 RTLIB::LRINT_F128,
2929 RTLIB::LRINT_PPCF128));
2930 break;
2931 case ISD::LLRINT:
2932 Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
2933 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
2934 RTLIB::LLRINT_F128,
2935 RTLIB::LLRINT_PPCF128));
2936 break;
2937 case ISD::VAARG:
2938 Results.push_back(DAG.expandVAArg(Node));
2939 Results.push_back(Results[0].getValue(1));
2940 break;
2941 case ISD::VACOPY:
2942 Results.push_back(DAG.expandVACopy(Node));
2943 break;
2944 case ISD::EXTRACT_VECTOR_ELT:
2945 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2946 // This must be an access of the only element. Return it.
2947 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2948 Node->getOperand(0));
2949 else
2950 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2951 Results.push_back(Tmp1);
2952 break;
2953 case ISD::EXTRACT_SUBVECTOR:
2954 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2955 break;
2956 case ISD::INSERT_SUBVECTOR:
2957 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2958 break;
2959 case ISD::CONCAT_VECTORS:
2960 Results.push_back(ExpandVectorBuildThroughStack(Node));
2961 break;
2962 case ISD::SCALAR_TO_VECTOR:
2963 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2964 break;
2965 case ISD::INSERT_VECTOR_ELT:
2966 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2967 Node->getOperand(1),
2968 Node->getOperand(2), dl));
2969 break;
2970 case ISD::VECTOR_SHUFFLE: {
2971 SmallVector<int, 32> NewMask;
2972 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2974 EVT VT = Node->getValueType(0);
2975 EVT EltVT = VT.getVectorElementType();
2976 SDValue Op0 = Node->getOperand(0);
2977 SDValue Op1 = Node->getOperand(1);
2978 if (!TLI.isTypeLegal(EltVT)) {
2979 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2981 // BUILD_VECTOR operands are allowed to be wider than the element type.
2982 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2983 // it.
2984 if (NewEltVT.bitsLT(EltVT)) {
2985 // Convert shuffle node.
2986 // If original node was v4i64 and the new EltVT is i32,
2987 // cast operands to v8i32 and re-build the mask.
2989 // Calculate new VT, the size of the new VT should be equal to original.
2990 EVT NewVT =
2991 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2992 VT.getSizeInBits() / NewEltVT.getSizeInBits());
2993 assert(NewVT.bitsEq(VT));
2995 // cast operands to new VT
2996 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2997 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2999 // Convert the shuffle mask
3000 unsigned int factor =
3001 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3003 // EltVT gets smaller
3004 assert(factor > 0);
3006 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3007 if (Mask[i] < 0) {
3008 for (unsigned fi = 0; fi < factor; ++fi)
3009 NewMask.push_back(Mask[i]);
3011 else {
3012 for (unsigned fi = 0; fi < factor; ++fi)
3013 NewMask.push_back(Mask[i]*factor+fi);
3016 Mask = NewMask;
3017 VT = NewVT;
3019 EltVT = NewEltVT;
3021 unsigned NumElems = VT.getVectorNumElements();
3022 SmallVector<SDValue, 16> Ops;
3023 for (unsigned i = 0; i != NumElems; ++i) {
3024 if (Mask[i] < 0) {
3025 Ops.push_back(DAG.getUNDEF(EltVT));
3026 continue;
3028 unsigned Idx = Mask[i];
3029 if (Idx < NumElems)
3030 Ops.push_back(DAG.getNode(
3031 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3032 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3033 else
3034 Ops.push_back(DAG.getNode(
3035 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3036 DAG.getConstant(Idx - NumElems, dl,
3037 TLI.getVectorIdxTy(DAG.getDataLayout()))));
3040 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3041 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3042 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3043 Results.push_back(Tmp1);
3044 break;
3046 case ISD::EXTRACT_ELEMENT: {
3047 EVT OpTy = Node->getOperand(0).getValueType();
3048 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3049 // 1 -> Hi
3050 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3051 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3052 TLI.getShiftAmountTy(
3053 Node->getOperand(0).getValueType(),
3054 DAG.getDataLayout())));
3055 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3056 } else {
3057 // 0 -> Lo
3058 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3059 Node->getOperand(0));
3061 Results.push_back(Tmp1);
3062 break;
3064 case ISD::STACKSAVE:
3065 // Expand to CopyFromReg if the target set
3066 // StackPointerRegisterToSaveRestore.
3067 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3068 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3069 Node->getValueType(0)));
3070 Results.push_back(Results[0].getValue(1));
3071 } else {
3072 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3073 Results.push_back(Node->getOperand(0));
3075 break;
3076 case ISD::STACKRESTORE:
3077 // Expand to CopyToReg if the target set
3078 // StackPointerRegisterToSaveRestore.
3079 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3080 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3081 Node->getOperand(1)));
3082 } else {
3083 Results.push_back(Node->getOperand(0));
3085 break;
3086 case ISD::GET_DYNAMIC_AREA_OFFSET:
3087 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3088 Results.push_back(Results[0].getValue(0));
3089 break;
3090 case ISD::FCOPYSIGN:
3091 Results.push_back(ExpandFCOPYSIGN(Node));
3092 break;
3093 case ISD::FNEG:
3094 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3095 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3096 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3097 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3098 Node->getOperand(0));
3099 Results.push_back(Tmp1);
3100 break;
3101 case ISD::FABS:
3102 Results.push_back(ExpandFABS(Node));
3103 break;
3104 case ISD::SMIN:
3105 case ISD::SMAX:
3106 case ISD::UMIN:
3107 case ISD::UMAX: {
3108 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3109 ISD::CondCode Pred;
3110 switch (Node->getOpcode()) {
3111 default: llvm_unreachable("How did we get here?");
3112 case ISD::SMAX: Pred = ISD::SETGT; break;
3113 case ISD::SMIN: Pred = ISD::SETLT; break;
3114 case ISD::UMAX: Pred = ISD::SETUGT; break;
3115 case ISD::UMIN: Pred = ISD::SETULT; break;
3117 Tmp1 = Node->getOperand(0);
3118 Tmp2 = Node->getOperand(1);
3119 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3120 Results.push_back(Tmp1);
3121 break;
3123 case ISD::FMINNUM:
3124 case ISD::FMAXNUM: {
3125 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3126 Results.push_back(Expanded);
3127 break;
3129 case ISD::FSIN:
3130 case ISD::FCOS: {
3131 EVT VT = Node->getValueType(0);
3132 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3133 // fcos which share the same operand and both are used.
3134 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3135 isSinCosLibcallAvailable(Node, TLI))
3136 && useSinCos(Node)) {
3137 SDVTList VTs = DAG.getVTList(VT, VT);
3138 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3139 if (Node->getOpcode() == ISD::FCOS)
3140 Tmp1 = Tmp1.getValue(1);
3141 Results.push_back(Tmp1);
3143 break;
3145 case ISD::FMAD:
3146 llvm_unreachable("Illegal fmad should never be formed");
3148 case ISD::FP16_TO_FP:
3149 if (Node->getValueType(0) != MVT::f32) {
3150 // We can extend to types bigger than f32 in two steps without changing
3151 // the result. Since "f16 -> f32" is much more commonly available, give
3152 // CodeGen the option of emitting that before resorting to a libcall.
3153 SDValue Res =
3154 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3155 Results.push_back(
3156 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3158 break;
3159 case ISD::FP_TO_FP16:
3160 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3161 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3162 SDValue Op = Node->getOperand(0);
3163 MVT SVT = Op.getSimpleValueType();
3164 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3165 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3166 // Under fastmath, we can expand this node into a fround followed by
3167 // a float-half conversion.
3168 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3169 DAG.getIntPtrConstant(0, dl));
3170 Results.push_back(
3171 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3174 break;
3175 case ISD::ConstantFP: {
3176 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3177 // Check to see if this FP immediate is already legal.
3178 // If this is a legal constant, turn it into a TargetConstantFP node.
3179 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3180 DAG.getMachineFunction().getFunction().hasOptSize()))
3181 Results.push_back(ExpandConstantFP(CFP, true));
3182 break;
3184 case ISD::Constant: {
3185 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3186 Results.push_back(ExpandConstant(CP));
3187 break;
3189 case ISD::FSUB: {
3190 EVT VT = Node->getValueType(0);
3191 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3192 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3193 const SDNodeFlags Flags = Node->getFlags();
3194 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3195 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3196 Results.push_back(Tmp1);
3198 break;
3200 case ISD::SUB: {
3201 EVT VT = Node->getValueType(0);
3202 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3203 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3204 "Don't know how to expand this subtraction!");
3205 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3206 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3207 VT));
3208 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3209 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3210 break;
3212 case ISD::UREM:
3213 case ISD::SREM: {
3214 EVT VT = Node->getValueType(0);
3215 bool isSigned = Node->getOpcode() == ISD::SREM;
3216 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3217 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3218 Tmp2 = Node->getOperand(0);
3219 Tmp3 = Node->getOperand(1);
3220 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3221 SDVTList VTs = DAG.getVTList(VT, VT);
3222 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3223 Results.push_back(Tmp1);
3224 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3225 // X % Y -> X-X/Y*Y
3226 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3227 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3228 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3229 Results.push_back(Tmp1);
3231 break;
3233 case ISD::UDIV:
3234 case ISD::SDIV: {
3235 bool isSigned = Node->getOpcode() == ISD::SDIV;
3236 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3237 EVT VT = Node->getValueType(0);
3238 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3239 SDVTList VTs = DAG.getVTList(VT, VT);
3240 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3241 Node->getOperand(1));
3242 Results.push_back(Tmp1);
3244 break;
3246 case ISD::MULHU:
3247 case ISD::MULHS: {
3248 unsigned ExpandOpcode =
3249 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3250 EVT VT = Node->getValueType(0);
3251 SDVTList VTs = DAG.getVTList(VT, VT);
3253 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3254 Node->getOperand(1));
3255 Results.push_back(Tmp1.getValue(1));
3256 break;
3258 case ISD::UMUL_LOHI:
3259 case ISD::SMUL_LOHI: {
3260 SDValue LHS = Node->getOperand(0);
3261 SDValue RHS = Node->getOperand(1);
3262 MVT VT = LHS.getSimpleValueType();
3263 unsigned MULHOpcode =
3264 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3266 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3267 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3268 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3269 break;
3272 SmallVector<SDValue, 4> Halves;
3273 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3274 assert(TLI.isTypeLegal(HalfType));
3275 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3276 HalfType, DAG,
3277 TargetLowering::MulExpansionKind::Always)) {
3278 for (unsigned i = 0; i < 2; ++i) {
3279 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3280 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3281 SDValue Shift = DAG.getConstant(
3282 HalfType.getScalarSizeInBits(), dl,
3283 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3284 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3285 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3287 break;
3289 break;
3291 case ISD::MUL: {
3292 EVT VT = Node->getValueType(0);
3293 SDVTList VTs = DAG.getVTList(VT, VT);
3294 // See if multiply or divide can be lowered using two-result operations.
3295 // We just need the low half of the multiply; try both the signed
3296 // and unsigned forms. If the target supports both SMUL_LOHI and
3297 // UMUL_LOHI, form a preference by checking which forms of plain
3298 // MULH it supports.
3299 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3300 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3301 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3302 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3303 unsigned OpToUse = 0;
3304 if (HasSMUL_LOHI && !HasMULHS) {
3305 OpToUse = ISD::SMUL_LOHI;
3306 } else if (HasUMUL_LOHI && !HasMULHU) {
3307 OpToUse = ISD::UMUL_LOHI;
3308 } else if (HasSMUL_LOHI) {
3309 OpToUse = ISD::SMUL_LOHI;
3310 } else if (HasUMUL_LOHI) {
3311 OpToUse = ISD::UMUL_LOHI;
3313 if (OpToUse) {
3314 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3315 Node->getOperand(1)));
3316 break;
3319 SDValue Lo, Hi;
3320 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3321 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3322 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3323 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3324 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3325 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3326 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3327 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3328 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3329 SDValue Shift =
3330 DAG.getConstant(HalfType.getSizeInBits(), dl,
3331 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3332 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3333 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3335 break;
3337 case ISD::FSHL:
3338 case ISD::FSHR:
3339 if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3340 Results.push_back(Tmp1);
3341 break;
3342 case ISD::ROTL:
3343 case ISD::ROTR:
3344 if (TLI.expandROT(Node, Tmp1, DAG))
3345 Results.push_back(Tmp1);
3346 break;
3347 case ISD::SADDSAT:
3348 case ISD::UADDSAT:
3349 case ISD::SSUBSAT:
3350 case ISD::USUBSAT:
3351 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3352 break;
3353 case ISD::SMULFIX:
3354 case ISD::SMULFIXSAT:
3355 case ISD::UMULFIX:
3356 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3357 break;
3358 case ISD::ADDCARRY:
3359 case ISD::SUBCARRY: {
3360 SDValue LHS = Node->getOperand(0);
3361 SDValue RHS = Node->getOperand(1);
3362 SDValue Carry = Node->getOperand(2);
3364 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3366 // Initial add of the 2 operands.
3367 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3368 EVT VT = LHS.getValueType();
3369 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3371 // Initial check for overflow.
3372 EVT CarryType = Node->getValueType(1);
3373 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3374 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3375 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3377 // Add of the sum and the carry.
3378 SDValue CarryExt =
3379 DAG.getZeroExtendInReg(DAG.getZExtOrTrunc(Carry, dl, VT), dl, MVT::i1);
3380 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3382 // Second check for overflow. If we are adding, we can only overflow if the
3383 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3384 // If we are subtracting, we can only overflow if the initial sum is 0 and
3385 // the carry is set, resulting in a new sum of all 1s.
3386 SDValue Zero = DAG.getConstant(0, dl, VT);
3387 SDValue Overflow2 =
3388 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3389 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3390 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3391 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3393 SDValue ResultCarry =
3394 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3396 Results.push_back(Sum2);
3397 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3398 break;
3400 case ISD::SADDO:
3401 case ISD::SSUBO: {
3402 SDValue Result, Overflow;
3403 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3404 Results.push_back(Result);
3405 Results.push_back(Overflow);
3406 break;
3408 case ISD::UADDO:
3409 case ISD::USUBO: {
3410 SDValue Result, Overflow;
3411 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3412 Results.push_back(Result);
3413 Results.push_back(Overflow);
3414 break;
3416 case ISD::UMULO:
3417 case ISD::SMULO: {
3418 SDValue Result, Overflow;
3419 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3420 Results.push_back(Result);
3421 Results.push_back(Overflow);
3423 break;
3425 case ISD::BUILD_PAIR: {
3426 EVT PairTy = Node->getValueType(0);
3427 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3428 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3429 Tmp2 = DAG.getNode(
3430 ISD::SHL, dl, PairTy, Tmp2,
3431 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3432 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3433 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3434 break;
3436 case ISD::SELECT:
3437 Tmp1 = Node->getOperand(0);
3438 Tmp2 = Node->getOperand(1);
3439 Tmp3 = Node->getOperand(2);
3440 if (Tmp1.getOpcode() == ISD::SETCC) {
3441 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3442 Tmp2, Tmp3,
3443 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3444 } else {
3445 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3446 DAG.getConstant(0, dl, Tmp1.getValueType()),
3447 Tmp2, Tmp3, ISD::SETNE);
3449 Tmp1->setFlags(Node->getFlags());
3450 Results.push_back(Tmp1);
3451 break;
3452 case ISD::BR_JT: {
3453 SDValue Chain = Node->getOperand(0);
3454 SDValue Table = Node->getOperand(1);
3455 SDValue Index = Node->getOperand(2);
3457 const DataLayout &TD = DAG.getDataLayout();
3458 EVT PTy = TLI.getPointerTy(TD);
3460 unsigned EntrySize =
3461 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3463 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3464 // This transformation needs to be done here since otherwise the MIPS
3465 // backend will end up emitting a three instruction multiply sequence
3466 // instead of a single shift and MSP430 will call a runtime function.
3467 if (llvm::isPowerOf2_32(EntrySize))
3468 Index = DAG.getNode(
3469 ISD::SHL, dl, Index.getValueType(), Index,
3470 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3471 else
3472 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3473 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3474 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3475 Index, Table);
3477 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3478 SDValue LD = DAG.getExtLoad(
3479 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3480 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3481 Addr = LD;
3482 if (TLI.isJumpTableRelative()) {
3483 // For PIC, the sequence is:
3484 // BRIND(load(Jumptable + index) + RelocBase)
3485 // RelocBase can be JumpTable, GOT or some sort of global base.
3486 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3487 TLI.getPICJumpTableRelocBase(Table, DAG));
3490 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3491 Results.push_back(Tmp1);
3492 break;
3494 case ISD::BRCOND:
3495 // Expand brcond's setcc into its constituent parts and create a BR_CC
3496 // Node.
3497 Tmp1 = Node->getOperand(0);
3498 Tmp2 = Node->getOperand(1);
3499 if (Tmp2.getOpcode() == ISD::SETCC) {
3500 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3501 Tmp1, Tmp2.getOperand(2),
3502 Tmp2.getOperand(0), Tmp2.getOperand(1),
3503 Node->getOperand(2));
3504 } else {
3505 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3506 if (Tmp2.isUndef() ||
3507 (Tmp2.getOpcode() == ISD::AND &&
3508 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3509 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3510 Tmp3 = Tmp2;
3511 else
3512 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3513 DAG.getConstant(1, dl, Tmp2.getValueType()));
3514 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3515 DAG.getCondCode(ISD::SETNE), Tmp3,
3516 DAG.getConstant(0, dl, Tmp3.getValueType()),
3517 Node->getOperand(2));
3519 Results.push_back(Tmp1);
3520 break;
3521 case ISD::SETCC: {
3522 Tmp1 = Node->getOperand(0);
3523 Tmp2 = Node->getOperand(1);
3524 Tmp3 = Node->getOperand(2);
3525 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3526 Tmp3, NeedInvert, dl);
3528 if (Legalized) {
3529 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3530 // condition code, create a new SETCC node.
3531 if (Tmp3.getNode())
3532 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3533 Tmp1, Tmp2, Tmp3, Node->getFlags());
3535 // If we expanded the SETCC by inverting the condition code, then wrap
3536 // the existing SETCC in a NOT to restore the intended condition.
3537 if (NeedInvert)
3538 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3540 Results.push_back(Tmp1);
3541 break;
3544 // Otherwise, SETCC for the given comparison type must be completely
3545 // illegal; expand it into a SELECT_CC.
3546 EVT VT = Node->getValueType(0);
3547 int TrueValue;
3548 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3549 case TargetLowering::ZeroOrOneBooleanContent:
3550 case TargetLowering::UndefinedBooleanContent:
3551 TrueValue = 1;
3552 break;
3553 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3554 TrueValue = -1;
3555 break;
3557 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3558 DAG.getConstant(TrueValue, dl, VT),
3559 DAG.getConstant(0, dl, VT),
3560 Tmp3);
3561 Tmp1->setFlags(Node->getFlags());
3562 Results.push_back(Tmp1);
3563 break;
3565 case ISD::SELECT_CC: {
3566 Tmp1 = Node->getOperand(0); // LHS
3567 Tmp2 = Node->getOperand(1); // RHS
3568 Tmp3 = Node->getOperand(2); // True
3569 Tmp4 = Node->getOperand(3); // False
3570 EVT VT = Node->getValueType(0);
3571 SDValue CC = Node->getOperand(4);
3572 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3574 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3575 // If the condition code is legal, then we need to expand this
3576 // node using SETCC and SELECT.
3577 EVT CmpVT = Tmp1.getValueType();
3578 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3579 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3580 "expanded.");
3581 EVT CCVT = getSetCCResultType(CmpVT);
3582 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3583 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3584 break;
3587 // SELECT_CC is legal, so the condition code must not be.
3588 bool Legalized = false;
3589 // Try to legalize by inverting the condition. This is for targets that
3590 // might support an ordered version of a condition, but not the unordered
3591 // version (or vice versa).
3592 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3593 Tmp1.getValueType().isInteger());
3594 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3595 // Use the new condition code and swap true and false
3596 Legalized = true;
3597 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3598 Tmp1->setFlags(Node->getFlags());
3599 } else {
3600 // If The inverse is not legal, then try to swap the arguments using
3601 // the inverse condition code.
3602 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3603 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3604 // The swapped inverse condition is legal, so swap true and false,
3605 // lhs and rhs.
3606 Legalized = true;
3607 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3608 Tmp1->setFlags(Node->getFlags());
3612 if (!Legalized) {
3613 Legalized = LegalizeSetCCCondCode(
3614 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3615 dl);
3617 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3619 // If we expanded the SETCC by inverting the condition code, then swap
3620 // the True/False operands to match.
3621 if (NeedInvert)
3622 std::swap(Tmp3, Tmp4);
3624 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3625 // condition code, create a new SELECT_CC node.
3626 if (CC.getNode()) {
3627 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3628 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3629 } else {
3630 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3631 CC = DAG.getCondCode(ISD::SETNE);
3632 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3633 Tmp2, Tmp3, Tmp4, CC);
3635 Tmp1->setFlags(Node->getFlags());
3637 Results.push_back(Tmp1);
3638 break;
3640 case ISD::BR_CC: {
3641 Tmp1 = Node->getOperand(0); // Chain
3642 Tmp2 = Node->getOperand(2); // LHS
3643 Tmp3 = Node->getOperand(3); // RHS
3644 Tmp4 = Node->getOperand(1); // CC
3646 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3647 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3648 (void)Legalized;
3649 assert(Legalized && "Can't legalize BR_CC with legal condition!");
3651 assert(!NeedInvert && "Don't know how to invert BR_CC!");
3653 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3654 // node.
3655 if (Tmp4.getNode()) {
3656 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3657 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3658 } else {
3659 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3660 Tmp4 = DAG.getCondCode(ISD::SETNE);
3661 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3662 Tmp2, Tmp3, Node->getOperand(4));
3664 Results.push_back(Tmp1);
3665 break;
3667 case ISD::BUILD_VECTOR:
3668 Results.push_back(ExpandBUILD_VECTOR(Node));
3669 break;
3670 case ISD::SRA:
3671 case ISD::SRL:
3672 case ISD::SHL: {
3673 // Scalarize vector SRA/SRL/SHL.
3674 EVT VT = Node->getValueType(0);
3675 assert(VT.isVector() && "Unable to legalize non-vector shift");
3676 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3677 unsigned NumElem = VT.getVectorNumElements();
3679 SmallVector<SDValue, 8> Scalars;
3680 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3681 SDValue Ex = DAG.getNode(
3682 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3683 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3684 SDValue Sh = DAG.getNode(
3685 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3686 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3687 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3688 VT.getScalarType(), Ex, Sh));
3691 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3692 ReplaceNode(SDValue(Node, 0), Result);
3693 break;
3695 case ISD::VECREDUCE_FADD:
3696 case ISD::VECREDUCE_FMUL:
3697 case ISD::VECREDUCE_ADD:
3698 case ISD::VECREDUCE_MUL:
3699 case ISD::VECREDUCE_AND:
3700 case ISD::VECREDUCE_OR:
3701 case ISD::VECREDUCE_XOR:
3702 case ISD::VECREDUCE_SMAX:
3703 case ISD::VECREDUCE_SMIN:
3704 case ISD::VECREDUCE_UMAX:
3705 case ISD::VECREDUCE_UMIN:
3706 case ISD::VECREDUCE_FMAX:
3707 case ISD::VECREDUCE_FMIN:
3708 Results.push_back(TLI.expandVecReduce(Node, DAG));
3709 break;
3710 case ISD::GLOBAL_OFFSET_TABLE:
3711 case ISD::GlobalAddress:
3712 case ISD::GlobalTLSAddress:
3713 case ISD::ExternalSymbol:
3714 case ISD::ConstantPool:
3715 case ISD::JumpTable:
3716 case ISD::INTRINSIC_W_CHAIN:
3717 case ISD::INTRINSIC_WO_CHAIN:
3718 case ISD::INTRINSIC_VOID:
3719 // FIXME: Custom lowering for these operations shouldn't return null!
3720 break;
3723 if (Results.empty() && Node->isStrictFPOpcode()) {
3724 // FIXME: We were asked to expand a strict floating-point operation,
3725 // but there is currently no expansion implemented that would preserve
3726 // the "strict" properties. For now, we just fall back to the non-strict
3727 // version if that is legal on the target. The actual mutation of the
3728 // operation will happen in SelectionDAGISel::DoInstructionSelection.
3729 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3730 Node->getValueType(0))
3731 == TargetLowering::Legal)
3732 return true;
3735 // Replace the original node with the legalized result.
3736 if (Results.empty()) {
3737 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3738 return false;
3741 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3742 ReplaceNode(Node, Results.data());
3743 return true;
3746 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3747 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3748 SmallVector<SDValue, 8> Results;
3749 SDLoc dl(Node);
3750 // FIXME: Check flags on the node to see if we can use a finite call.
3751 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3752 unsigned Opc = Node->getOpcode();
3753 switch (Opc) {
3754 case ISD::ATOMIC_FENCE: {
3755 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3756 // FIXME: handle "fence singlethread" more efficiently.
3757 TargetLowering::ArgListTy Args;
3759 TargetLowering::CallLoweringInfo CLI(DAG);
3760 CLI.setDebugLoc(dl)
3761 .setChain(Node->getOperand(0))
3762 .setLibCallee(
3763 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3764 DAG.getExternalSymbol("__sync_synchronize",
3765 TLI.getPointerTy(DAG.getDataLayout())),
3766 std::move(Args));
3768 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3770 Results.push_back(CallResult.second);
3771 break;
3773 // By default, atomic intrinsics are marked Legal and lowered. Targets
3774 // which don't support them directly, however, may want libcalls, in which
3775 // case they mark them Expand, and we get here.
3776 case ISD::ATOMIC_SWAP:
3777 case ISD::ATOMIC_LOAD_ADD:
3778 case ISD::ATOMIC_LOAD_SUB:
3779 case ISD::ATOMIC_LOAD_AND:
3780 case ISD::ATOMIC_LOAD_CLR:
3781 case ISD::ATOMIC_LOAD_OR:
3782 case ISD::ATOMIC_LOAD_XOR:
3783 case ISD::ATOMIC_LOAD_NAND:
3784 case ISD::ATOMIC_LOAD_MIN:
3785 case ISD::ATOMIC_LOAD_MAX:
3786 case ISD::ATOMIC_LOAD_UMIN:
3787 case ISD::ATOMIC_LOAD_UMAX:
3788 case ISD::ATOMIC_CMP_SWAP: {
3789 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3790 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3791 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
3793 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3794 Results.push_back(Tmp.first);
3795 Results.push_back(Tmp.second);
3796 break;
3798 case ISD::TRAP: {
3799 // If this operation is not supported, lower it to 'abort()' call
3800 TargetLowering::ArgListTy Args;
3801 TargetLowering::CallLoweringInfo CLI(DAG);
3802 CLI.setDebugLoc(dl)
3803 .setChain(Node->getOperand(0))
3804 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3805 DAG.getExternalSymbol(
3806 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3807 std::move(Args));
3808 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3810 Results.push_back(CallResult.second);
3811 break;
3813 case ISD::FMINNUM:
3814 case ISD::STRICT_FMINNUM:
3815 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3816 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3817 RTLIB::FMIN_PPCF128));
3818 break;
3819 case ISD::FMAXNUM:
3820 case ISD::STRICT_FMAXNUM:
3821 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3822 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3823 RTLIB::FMAX_PPCF128));
3824 break;
3825 case ISD::FSQRT:
3826 case ISD::STRICT_FSQRT:
3827 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3828 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3829 RTLIB::SQRT_PPCF128));
3830 break;
3831 case ISD::FCBRT:
3832 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3833 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3834 RTLIB::CBRT_PPCF128));
3835 break;
3836 case ISD::FSIN:
3837 case ISD::STRICT_FSIN:
3838 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3839 RTLIB::SIN_F80, RTLIB::SIN_F128,
3840 RTLIB::SIN_PPCF128));
3841 break;
3842 case ISD::FCOS:
3843 case ISD::STRICT_FCOS:
3844 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3845 RTLIB::COS_F80, RTLIB::COS_F128,
3846 RTLIB::COS_PPCF128));
3847 break;
3848 case ISD::FSINCOS:
3849 // Expand into sincos libcall.
3850 ExpandSinCosLibCall(Node, Results);
3851 break;
3852 case ISD::FLOG:
3853 case ISD::STRICT_FLOG:
3854 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
3855 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3856 RTLIB::LOG_FINITE_F64,
3857 RTLIB::LOG_FINITE_F80,
3858 RTLIB::LOG_FINITE_F128,
3859 RTLIB::LOG_FINITE_PPCF128));
3860 else
3861 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3862 RTLIB::LOG_F80, RTLIB::LOG_F128,
3863 RTLIB::LOG_PPCF128));
3864 break;
3865 case ISD::FLOG2:
3866 case ISD::STRICT_FLOG2:
3867 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
3868 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3869 RTLIB::LOG2_FINITE_F64,
3870 RTLIB::LOG2_FINITE_F80,
3871 RTLIB::LOG2_FINITE_F128,
3872 RTLIB::LOG2_FINITE_PPCF128));
3873 else
3874 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3875 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3876 RTLIB::LOG2_PPCF128));
3877 break;
3878 case ISD::FLOG10:
3879 case ISD::STRICT_FLOG10:
3880 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
3881 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3882 RTLIB::LOG10_FINITE_F64,
3883 RTLIB::LOG10_FINITE_F80,
3884 RTLIB::LOG10_FINITE_F128,
3885 RTLIB::LOG10_FINITE_PPCF128));
3886 else
3887 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3888 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3889 RTLIB::LOG10_PPCF128));
3890 break;
3891 case ISD::FEXP:
3892 case ISD::STRICT_FEXP:
3893 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
3894 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3895 RTLIB::EXP_FINITE_F64,
3896 RTLIB::EXP_FINITE_F80,
3897 RTLIB::EXP_FINITE_F128,
3898 RTLIB::EXP_FINITE_PPCF128));
3899 else
3900 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3901 RTLIB::EXP_F80, RTLIB::EXP_F128,
3902 RTLIB::EXP_PPCF128));
3903 break;
3904 case ISD::FEXP2:
3905 case ISD::STRICT_FEXP2:
3906 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
3907 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3908 RTLIB::EXP2_FINITE_F64,
3909 RTLIB::EXP2_FINITE_F80,
3910 RTLIB::EXP2_FINITE_F128,
3911 RTLIB::EXP2_FINITE_PPCF128));
3912 else
3913 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3914 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3915 RTLIB::EXP2_PPCF128));
3916 break;
3917 case ISD::FTRUNC:
3918 case ISD::STRICT_FTRUNC:
3919 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3920 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3921 RTLIB::TRUNC_PPCF128));
3922 break;
3923 case ISD::FFLOOR:
3924 case ISD::STRICT_FFLOOR:
3925 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3926 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3927 RTLIB::FLOOR_PPCF128));
3928 break;
3929 case ISD::FCEIL:
3930 case ISD::STRICT_FCEIL:
3931 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3932 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3933 RTLIB::CEIL_PPCF128));
3934 break;
3935 case ISD::FRINT:
3936 case ISD::STRICT_FRINT:
3937 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3938 RTLIB::RINT_F80, RTLIB::RINT_F128,
3939 RTLIB::RINT_PPCF128));
3940 break;
3941 case ISD::FNEARBYINT:
3942 case ISD::STRICT_FNEARBYINT:
3943 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3944 RTLIB::NEARBYINT_F64,
3945 RTLIB::NEARBYINT_F80,
3946 RTLIB::NEARBYINT_F128,
3947 RTLIB::NEARBYINT_PPCF128));
3948 break;
3949 case ISD::FROUND:
3950 case ISD::STRICT_FROUND:
3951 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3952 RTLIB::ROUND_F64,
3953 RTLIB::ROUND_F80,
3954 RTLIB::ROUND_F128,
3955 RTLIB::ROUND_PPCF128));
3956 break;
3957 case ISD::FPOWI:
3958 case ISD::STRICT_FPOWI:
3959 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3960 RTLIB::POWI_F80, RTLIB::POWI_F128,
3961 RTLIB::POWI_PPCF128));
3962 break;
3963 case ISD::FPOW:
3964 case ISD::STRICT_FPOW:
3965 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
3966 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
3967 RTLIB::POW_FINITE_F64,
3968 RTLIB::POW_FINITE_F80,
3969 RTLIB::POW_FINITE_F128,
3970 RTLIB::POW_FINITE_PPCF128));
3971 else
3972 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3973 RTLIB::POW_F80, RTLIB::POW_F128,
3974 RTLIB::POW_PPCF128));
3975 break;
3976 case ISD::FDIV:
3977 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3978 RTLIB::DIV_F80, RTLIB::DIV_F128,
3979 RTLIB::DIV_PPCF128));
3980 break;
3981 case ISD::FREM:
3982 case ISD::STRICT_FREM:
3983 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3984 RTLIB::REM_F80, RTLIB::REM_F128,
3985 RTLIB::REM_PPCF128));
3986 break;
3987 case ISD::FMA:
3988 case ISD::STRICT_FMA:
3989 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
3990 RTLIB::FMA_F80, RTLIB::FMA_F128,
3991 RTLIB::FMA_PPCF128));
3992 break;
3993 case ISD::FADD:
3994 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
3995 RTLIB::ADD_F80, RTLIB::ADD_F128,
3996 RTLIB::ADD_PPCF128));
3997 break;
3998 case ISD::FMUL:
3999 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4000 RTLIB::MUL_F80, RTLIB::MUL_F128,
4001 RTLIB::MUL_PPCF128));
4002 break;
4003 case ISD::FP16_TO_FP:
4004 if (Node->getValueType(0) == MVT::f32) {
4005 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4007 break;
4008 case ISD::FP_TO_FP16: {
4009 RTLIB::Libcall LC =
4010 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4011 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4012 Results.push_back(ExpandLibCall(LC, Node, false));
4013 break;
4015 case ISD::FSUB:
4016 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4017 RTLIB::SUB_F80, RTLIB::SUB_F128,
4018 RTLIB::SUB_PPCF128));
4019 break;
4020 case ISD::SREM:
4021 Results.push_back(ExpandIntLibCall(Node, true,
4022 RTLIB::SREM_I8,
4023 RTLIB::SREM_I16, RTLIB::SREM_I32,
4024 RTLIB::SREM_I64, RTLIB::SREM_I128));
4025 break;
4026 case ISD::UREM:
4027 Results.push_back(ExpandIntLibCall(Node, false,
4028 RTLIB::UREM_I8,
4029 RTLIB::UREM_I16, RTLIB::UREM_I32,
4030 RTLIB::UREM_I64, RTLIB::UREM_I128));
4031 break;
4032 case ISD::SDIV:
4033 Results.push_back(ExpandIntLibCall(Node, true,
4034 RTLIB::SDIV_I8,
4035 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4036 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4037 break;
4038 case ISD::UDIV:
4039 Results.push_back(ExpandIntLibCall(Node, false,
4040 RTLIB::UDIV_I8,
4041 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4042 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4043 break;
4044 case ISD::SDIVREM:
4045 case ISD::UDIVREM:
4046 // Expand into divrem libcall
4047 ExpandDivRemLibCall(Node, Results);
4048 break;
4049 case ISD::MUL:
4050 Results.push_back(ExpandIntLibCall(Node, false,
4051 RTLIB::MUL_I8,
4052 RTLIB::MUL_I16, RTLIB::MUL_I32,
4053 RTLIB::MUL_I64, RTLIB::MUL_I128));
4054 break;
4055 case ISD::CTLZ_ZERO_UNDEF:
4056 switch (Node->getSimpleValueType(0).SimpleTy) {
4057 default:
4058 llvm_unreachable("LibCall explicitly requested, but not available");
4059 case MVT::i32:
4060 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4061 break;
4062 case MVT::i64:
4063 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4064 break;
4065 case MVT::i128:
4066 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4067 break;
4069 break;
4072 // Replace the original node with the legalized result.
4073 if (!Results.empty()) {
4074 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4075 ReplaceNode(Node, Results.data());
4076 } else
4077 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4080 // Determine the vector type to use in place of an original scalar element when
4081 // promoting equally sized vectors.
4082 static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4083 MVT EltVT, MVT NewEltVT) {
4084 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4085 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4086 assert(TLI.isTypeLegal(MidVT) && "unexpected");
4087 return MidVT;
4090 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4091 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4092 SmallVector<SDValue, 8> Results;
4093 MVT OVT = Node->getSimpleValueType(0);
4094 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4095 Node->getOpcode() == ISD::SINT_TO_FP ||
4096 Node->getOpcode() == ISD::SETCC ||
4097 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4098 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4099 OVT = Node->getOperand(0).getSimpleValueType();
4101 if (Node->getOpcode() == ISD::BR_CC)
4102 OVT = Node->getOperand(2).getSimpleValueType();
4103 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4104 SDLoc dl(Node);
4105 SDValue Tmp1, Tmp2, Tmp3;
4106 switch (Node->getOpcode()) {
4107 case ISD::CTTZ:
4108 case ISD::CTTZ_ZERO_UNDEF:
4109 case ISD::CTLZ:
4110 case ISD::CTLZ_ZERO_UNDEF:
4111 case ISD::CTPOP:
4112 // Zero extend the argument.
4113 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4114 if (Node->getOpcode() == ISD::CTTZ) {
4115 // The count is the same in the promoted type except if the original
4116 // value was zero. This can be handled by setting the bit just off
4117 // the top of the original type.
4118 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4119 OVT.getSizeInBits());
4120 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4121 DAG.getConstant(TopBit, dl, NVT));
4123 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4124 // already the correct result.
4125 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4126 if (Node->getOpcode() == ISD::CTLZ ||
4127 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4128 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4129 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4130 DAG.getConstant(NVT.getSizeInBits() -
4131 OVT.getSizeInBits(), dl, NVT));
4133 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4134 break;
4135 case ISD::BITREVERSE:
4136 case ISD::BSWAP: {
4137 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4138 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4139 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4140 Tmp1 = DAG.getNode(
4141 ISD::SRL, dl, NVT, Tmp1,
4142 DAG.getConstant(DiffBits, dl,
4143 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4145 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4146 break;
4148 case ISD::FP_TO_UINT:
4149 case ISD::FP_TO_SINT:
4150 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4151 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4152 Results.push_back(Tmp1);
4153 break;
4154 case ISD::UINT_TO_FP:
4155 case ISD::SINT_TO_FP:
4156 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4157 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4158 Results.push_back(Tmp1);
4159 break;
4160 case ISD::VAARG: {
4161 SDValue Chain = Node->getOperand(0); // Get the chain.
4162 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4164 unsigned TruncOp;
4165 if (OVT.isVector()) {
4166 TruncOp = ISD::BITCAST;
4167 } else {
4168 assert(OVT.isInteger()
4169 && "VAARG promotion is supported only for vectors or integer types");
4170 TruncOp = ISD::TRUNCATE;
4173 // Perform the larger operation, then convert back
4174 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4175 Node->getConstantOperandVal(3));
4176 Chain = Tmp1.getValue(1);
4178 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4180 // Modified the chain result - switch anything that used the old chain to
4181 // use the new one.
4182 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4183 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4184 if (UpdatedNodes) {
4185 UpdatedNodes->insert(Tmp2.getNode());
4186 UpdatedNodes->insert(Chain.getNode());
4188 ReplacedNode(Node);
4189 break;
4191 case ISD::MUL:
4192 case ISD::SDIV:
4193 case ISD::SREM:
4194 case ISD::UDIV:
4195 case ISD::UREM:
4196 case ISD::AND:
4197 case ISD::OR:
4198 case ISD::XOR: {
4199 unsigned ExtOp, TruncOp;
4200 if (OVT.isVector()) {
4201 ExtOp = ISD::BITCAST;
4202 TruncOp = ISD::BITCAST;
4203 } else {
4204 assert(OVT.isInteger() && "Cannot promote logic operation");
4206 switch (Node->getOpcode()) {
4207 default:
4208 ExtOp = ISD::ANY_EXTEND;
4209 break;
4210 case ISD::SDIV:
4211 case ISD::SREM:
4212 ExtOp = ISD::SIGN_EXTEND;
4213 break;
4214 case ISD::UDIV:
4215 case ISD::UREM:
4216 ExtOp = ISD::ZERO_EXTEND;
4217 break;
4219 TruncOp = ISD::TRUNCATE;
4221 // Promote each of the values to the new type.
4222 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4223 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4224 // Perform the larger operation, then convert back
4225 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4226 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4227 break;
4229 case ISD::UMUL_LOHI:
4230 case ISD::SMUL_LOHI: {
4231 // Promote to a multiply in a wider integer type.
4232 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4233 : ISD::SIGN_EXTEND;
4234 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4235 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4236 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4238 auto &DL = DAG.getDataLayout();
4239 unsigned OriginalSize = OVT.getScalarSizeInBits();
4240 Tmp2 = DAG.getNode(
4241 ISD::SRL, dl, NVT, Tmp1,
4242 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4243 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4244 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4245 break;
4247 case ISD::SELECT: {
4248 unsigned ExtOp, TruncOp;
4249 if (Node->getValueType(0).isVector() ||
4250 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4251 ExtOp = ISD::BITCAST;
4252 TruncOp = ISD::BITCAST;
4253 } else if (Node->getValueType(0).isInteger()) {
4254 ExtOp = ISD::ANY_EXTEND;
4255 TruncOp = ISD::TRUNCATE;
4256 } else {
4257 ExtOp = ISD::FP_EXTEND;
4258 TruncOp = ISD::FP_ROUND;
4260 Tmp1 = Node->getOperand(0);
4261 // Promote each of the values to the new type.
4262 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4263 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4264 // Perform the larger operation, then round down.
4265 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4266 Tmp1->setFlags(Node->getFlags());
4267 if (TruncOp != ISD::FP_ROUND)
4268 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4269 else
4270 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4271 DAG.getIntPtrConstant(0, dl));
4272 Results.push_back(Tmp1);
4273 break;
4275 case ISD::VECTOR_SHUFFLE: {
4276 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4278 // Cast the two input vectors.
4279 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4280 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4282 // Convert the shuffle mask to the right # elements.
4283 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4284 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4285 Results.push_back(Tmp1);
4286 break;
4288 case ISD::SETCC: {
4289 unsigned ExtOp = ISD::FP_EXTEND;
4290 if (NVT.isInteger()) {
4291 ISD::CondCode CCCode =
4292 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4293 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4295 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4296 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4297 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4298 Tmp2, Node->getOperand(2), Node->getFlags()));
4299 break;
4301 case ISD::BR_CC: {
4302 unsigned ExtOp = ISD::FP_EXTEND;
4303 if (NVT.isInteger()) {
4304 ISD::CondCode CCCode =
4305 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4306 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4308 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4309 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4310 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4311 Node->getOperand(0), Node->getOperand(1),
4312 Tmp1, Tmp2, Node->getOperand(4)));
4313 break;
4315 case ISD::FADD:
4316 case ISD::FSUB:
4317 case ISD::FMUL:
4318 case ISD::FDIV:
4319 case ISD::FREM:
4320 case ISD::FMINNUM:
4321 case ISD::FMAXNUM:
4322 case ISD::FPOW:
4323 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4324 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4325 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4326 Node->getFlags());
4327 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4328 Tmp3, DAG.getIntPtrConstant(0, dl)));
4329 break;
4330 case ISD::FMA:
4331 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4332 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4333 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4334 Results.push_back(
4335 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4336 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4337 DAG.getIntPtrConstant(0, dl)));
4338 break;
4339 case ISD::FCOPYSIGN:
4340 case ISD::FPOWI: {
4341 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4342 Tmp2 = Node->getOperand(1);
4343 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4345 // fcopysign doesn't change anything but the sign bit, so
4346 // (fp_round (fcopysign (fpext a), b))
4347 // is as precise as
4348 // (fp_round (fpext a))
4349 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4350 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4351 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4352 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4353 break;
4355 case ISD::FFLOOR:
4356 case ISD::FCEIL:
4357 case ISD::FRINT:
4358 case ISD::FNEARBYINT:
4359 case ISD::FROUND:
4360 case ISD::FTRUNC:
4361 case ISD::FNEG:
4362 case ISD::FSQRT:
4363 case ISD::FSIN:
4364 case ISD::FCOS:
4365 case ISD::FLOG:
4366 case ISD::FLOG2:
4367 case ISD::FLOG10:
4368 case ISD::FABS:
4369 case ISD::FEXP:
4370 case ISD::FEXP2:
4371 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4372 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4373 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4374 Tmp2, DAG.getIntPtrConstant(0, dl)));
4375 break;
4376 case ISD::BUILD_VECTOR: {
4377 MVT EltVT = OVT.getVectorElementType();
4378 MVT NewEltVT = NVT.getVectorElementType();
4380 // Handle bitcasts to a different vector type with the same total bit size
4382 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4383 // =>
4384 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4386 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4387 "Invalid promote type for build_vector");
4388 assert(NewEltVT.bitsLT(EltVT) && "not handled");
4390 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4392 SmallVector<SDValue, 8> NewOps;
4393 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4394 SDValue Op = Node->getOperand(I);
4395 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4398 SDLoc SL(Node);
4399 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4400 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4401 Results.push_back(CvtVec);
4402 break;
4404 case ISD::EXTRACT_VECTOR_ELT: {
4405 MVT EltVT = OVT.getVectorElementType();
4406 MVT NewEltVT = NVT.getVectorElementType();
4408 // Handle bitcasts to a different vector type with the same total bit size.
4410 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4411 // =>
4412 // v4i32:castx = bitcast x:v2i64
4414 // i64 = bitcast
4415 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4416 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4419 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4420 "Invalid promote type for extract_vector_elt");
4421 assert(NewEltVT.bitsLT(EltVT) && "not handled");
4423 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4424 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4426 SDValue Idx = Node->getOperand(1);
4427 EVT IdxVT = Idx.getValueType();
4428 SDLoc SL(Node);
4429 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4430 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4432 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4434 SmallVector<SDValue, 8> NewOps;
4435 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4436 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4437 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4439 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4440 CastVec, TmpIdx);
4441 NewOps.push_back(Elt);
4444 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4445 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4446 break;
4448 case ISD::INSERT_VECTOR_ELT: {
4449 MVT EltVT = OVT.getVectorElementType();
4450 MVT NewEltVT = NVT.getVectorElementType();
4452 // Handle bitcasts to a different vector type with the same total bit size
4454 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4455 // =>
4456 // v4i32:castx = bitcast x:v2i64
4457 // v2i32:casty = bitcast y:i64
4459 // v2i64 = bitcast
4460 // (v4i32 insert_vector_elt
4461 // (v4i32 insert_vector_elt v4i32:castx,
4462 // (extract_vector_elt casty, 0), 2 * z),
4463 // (extract_vector_elt casty, 1), (2 * z + 1))
4465 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4466 "Invalid promote type for insert_vector_elt");
4467 assert(NewEltVT.bitsLT(EltVT) && "not handled");
4469 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4470 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4472 SDValue Val = Node->getOperand(1);
4473 SDValue Idx = Node->getOperand(2);
4474 EVT IdxVT = Idx.getValueType();
4475 SDLoc SL(Node);
4477 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4478 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4480 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4481 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4483 SDValue NewVec = CastVec;
4484 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4485 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4486 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4488 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4489 CastVal, IdxOffset);
4491 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4492 NewVec, Elt, InEltIdx);
4495 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4496 break;
4498 case ISD::SCALAR_TO_VECTOR: {
4499 MVT EltVT = OVT.getVectorElementType();
4500 MVT NewEltVT = NVT.getVectorElementType();
4502 // Handle bitcasts to different vector type with the same total bit size.
4504 // e.g. v2i64 = scalar_to_vector x:i64
4505 // =>
4506 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4509 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4510 SDValue Val = Node->getOperand(0);
4511 SDLoc SL(Node);
4513 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4514 SDValue Undef = DAG.getUNDEF(MidVT);
4516 SmallVector<SDValue, 8> NewElts;
4517 NewElts.push_back(CastVal);
4518 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4519 NewElts.push_back(Undef);
4521 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4522 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4523 Results.push_back(CvtVec);
4524 break;
4526 case ISD::ATOMIC_SWAP: {
4527 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4528 SDLoc SL(Node);
4529 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4530 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
4531 "unexpected promotion type");
4532 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
4533 "unexpected atomic_swap with illegal type");
4535 SDValue NewAtomic
4536 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4537 DAG.getVTList(NVT, MVT::Other),
4538 { AM->getChain(), AM->getBasePtr(), CastVal },
4539 AM->getMemOperand());
4540 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4541 Results.push_back(NewAtomic.getValue(1));
4542 break;
4546 // Replace the original node with the legalized result.
4547 if (!Results.empty()) {
4548 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4549 ReplaceNode(Node, Results.data());
4550 } else
4551 LLVM_DEBUG(dbgs() << "Could not promote node\n");
4554 /// This is the entry point for the file.
4555 void SelectionDAG::Legalize() {
4556 AssignTopologicalOrder();
4558 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4559 // Use a delete listener to remove nodes which were deleted during
4560 // legalization from LegalizeNodes. This is needed to handle the situation
4561 // where a new node is allocated by the object pool to the same address of a
4562 // previously deleted node.
4563 DAGNodeDeletedListener DeleteListener(
4564 *this,
4565 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4567 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4569 // Visit all the nodes. We start in topological order, so that we see
4570 // nodes with their original operands intact. Legalization can produce
4571 // new nodes which may themselves need to be legalized. Iterate until all
4572 // nodes have been legalized.
4573 while (true) {
4574 bool AnyLegalized = false;
4575 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4576 --NI;
4578 SDNode *N = &*NI;
4579 if (N->use_empty() && N != getRoot().getNode()) {
4580 ++NI;
4581 DeleteNode(N);
4582 continue;
4585 if (LegalizedNodes.insert(N).second) {
4586 AnyLegalized = true;
4587 Legalizer.LegalizeOp(N);
4589 if (N->use_empty() && N != getRoot().getNode()) {
4590 ++NI;
4591 DeleteNode(N);
4595 if (!AnyLegalized)
4596 break;
4600 // Remove dead nodes now.
4601 RemoveDeadNodes();
4604 bool SelectionDAG::LegalizeOp(SDNode *N,
4605 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4606 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4607 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4609 // Directly insert the node in question, and legalize it. This will recurse
4610 // as needed through operands.
4611 LegalizedNodes.insert(N);
4612 Legalizer.LegalizeOp(N);
4614 return LegalizedNodes.count(N);