llvm/lib/CodeGen/GlobalISel/CallLowering.cpp

   1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements some simple delegations needed for call lowering.
  11 ///
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/CodeGen/Analysis.h"
  15 #include "llvm/CodeGen/CallingConvLower.h"
  16 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
  17 #include "llvm/CodeGen/GlobalISel/Utils.h"
  18 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  19 #include "llvm/CodeGen/MachineOperand.h"
  20 #include "llvm/CodeGen/MachineRegisterInfo.h"
  21 #include "llvm/CodeGen/TargetLowering.h"
  22 #include "llvm/IR/DataLayout.h"
  23 #include "llvm/IR/Instructions.h"
  24 #include "llvm/IR/LLVMContext.h"
  25 #include "llvm/IR/Module.h"
  26 #include "llvm/Target/TargetMachine.h"
  27
  28 #define DEBUG_TYPE "call-lowering"
  29
  30 using namespace llvm;
  31
  32 void CallLowering::anchor() {}
  33
  34 /// Helper function which updates \p Flags when \p AttrFn returns true.
  35 static void
  36 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
  37                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
  38   if (AttrFn(Attribute::SExt))
  39     Flags.setSExt();
  40   if (AttrFn(Attribute::ZExt))
  41     Flags.setZExt();
  42   if (AttrFn(Attribute::InReg))
  43     Flags.setInReg();
  44   if (AttrFn(Attribute::StructRet))
  45     Flags.setSRet();
  46   if (AttrFn(Attribute::Nest))
  47     Flags.setNest();
  48   if (AttrFn(Attribute::ByVal))
  49     Flags.setByVal();
  50   if (AttrFn(Attribute::Preallocated))
  51     Flags.setPreallocated();
  52   if (AttrFn(Attribute::InAlloca))
  53     Flags.setInAlloca();
  54   if (AttrFn(Attribute::Returned))
  55     Flags.setReturned();
  56   if (AttrFn(Attribute::SwiftSelf))
  57     Flags.setSwiftSelf();
  58   if (AttrFn(Attribute::SwiftAsync))
  59     Flags.setSwiftAsync();
  60   if (AttrFn(Attribute::SwiftError))
  61     Flags.setSwiftError();
  62 }
  63
  64 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
  65                                                      unsigned ArgIdx) const {
  66   ISD::ArgFlagsTy Flags;
  67   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
  68     return Call.paramHasAttr(ArgIdx, Attr);
  69   });
  70   return Flags;
  71 }
  72
  73 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
  74                                              const AttributeList &Attrs,
  75                                              unsigned OpIdx) const {
  76   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
  77     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
  78   });
  79 }
  80
  81 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
  82                              ArrayRef<Register> ResRegs,
  83                              ArrayRef<ArrayRef<Register>> ArgRegs,
  84                              Register SwiftErrorVReg,
  85                              std::function<unsigned()> GetCalleeReg) const {
  86   CallLoweringInfo Info;
  87   const DataLayout &DL = MIRBuilder.getDataLayout();
  88   MachineFunction &MF = MIRBuilder.getMF();
  89   MachineRegisterInfo &MRI = MF.getRegInfo();
  90   bool CanBeTailCalled = CB.isTailCall() &&
  91                          isInTailCallPosition(CB, MF.getTarget()) &&
  92                          (MF.getFunction()
  93                               .getFnAttribute("disable-tail-calls")
  94                               .getValueAsString() != "true");
  95
  96   CallingConv::ID CallConv = CB.getCallingConv();
  97   Type *RetTy = CB.getType();
  98   bool IsVarArg = CB.getFunctionType()->isVarArg();
  99
 100   SmallVector<BaseArgInfo, 4> SplitArgs;
 101   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
 102   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
 103
 104   if (!Info.CanLowerReturn) {
 105     // Callee requires sret demotion.
 106     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
 107
 108     // The sret demotion isn't compatible with tail-calls, since the sret
 109     // argument points into the caller's stack frame.
 110     CanBeTailCalled = false;
 111   }
 112
 113
 114   // First step is to marshall all the function's parameters into the correct
 115   // physregs and memory locations. Gather the sequence of argument types that
 116   // we'll pass to the assigner function.
 117   unsigned i = 0;
 118   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
 119   for (auto &Arg : CB.args()) {
 120     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
 121                     i < NumFixedArgs};
 122     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
 123
 124     // If we have an explicit sret argument that is an Instruction, (i.e., it
 125     // might point to function-local memory), we can't meaningfully tail-call.
 126     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
 127       CanBeTailCalled = false;
 128
 129     Info.OrigArgs.push_back(OrigArg);
 130     ++i;
 131   }
 132
 133   // Try looking through a bitcast from one function type to another.
 134   // Commonly happens with calls to objc_msgSend().
 135   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
 136   if (const Function *F = dyn_cast<Function>(CalleeV))
 137     Info.Callee = MachineOperand::CreateGA(F, 0);
 138   else
 139     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
 140
 141   Register ReturnHintAlignReg;
 142   Align ReturnHintAlign;
 143
 144   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, ISD::ArgFlagsTy{}};
 145
 146   if (!Info.OrigRet.Ty->isVoidTy()) {
 147     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
 148
 149     if (MaybeAlign Alignment = CB.getRetAlign()) {
 150       if (*Alignment > Align(1)) {
 151         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
 152         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
 153         ReturnHintAlign = *Alignment;
 154       }
 155     }
 156   }
 157
 158   Info.CB = &CB;
 159   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
 160   Info.CallConv = CallConv;
 161   Info.SwiftErrorVReg = SwiftErrorVReg;
 162   Info.IsMustTailCall = CB.isMustTailCall();
 163   Info.IsTailCall = CanBeTailCalled;
 164   Info.IsVarArg = IsVarArg;
 165   if (!lowerCall(MIRBuilder, Info))
 166     return false;
 167
 168   if (ReturnHintAlignReg && !Info.IsTailCall) {
 169     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
 170                                 ReturnHintAlign);
 171   }
 172
 173   return true;
 174 }
 175
 176 template <typename FuncInfoTy>
 177 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 178                                const DataLayout &DL,
 179                                const FuncInfoTy &FuncInfo) const {
 180   auto &Flags = Arg.Flags[0];
 181   const AttributeList &Attrs = FuncInfo.getAttributes();
 182   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
 183
 184   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
 185   if (PtrTy) {
 186     Flags.setPointer();
 187     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
 188   }
 189
 190   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
 191   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated()) {
 192     assert(OpIdx >= AttributeList::FirstArgIndex);
 193     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
 194
 195     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
 196     if (!ElementTy)
 197       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
 198     if (!ElementTy)
 199       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
 200     assert(ElementTy && "Must have byval, inalloca or preallocated type");
 201     Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
 202
 203     // For ByVal, alignment should be passed from FE.  BE will guess if
 204     // this info is not there but there are cases it cannot get right.
 205     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
 206       MemAlign = *ParamAlign;
 207     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
 208       MemAlign = *ParamAlign;
 209     else
 210       MemAlign = Align(getTLI()->getByValTypeAlignment(ElementTy, DL));
 211   } else if (OpIdx >= AttributeList::FirstArgIndex) {
 212     if (auto ParamAlign =
 213             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
 214       MemAlign = *ParamAlign;
 215   }
 216   Flags.setMemAlign(MemAlign);
 217   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
 218
 219   // Don't try to use the returned attribute if the argument is marked as
 220   // swiftself, since it won't be passed in x0.
 221   if (Flags.isSwiftSelf())
 222     Flags.setReturned(false);
 223 }
 224
 225 template void
 226 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 227                                     const DataLayout &DL,
 228                                     const Function &FuncInfo) const;
 229
 230 template void
 231 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 232                                     const DataLayout &DL,
 233                                     const CallBase &FuncInfo) const;
 234
 235 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
 236                                      SmallVectorImpl<ArgInfo> &SplitArgs,
 237                                      const DataLayout &DL,
 238                                      CallingConv::ID CallConv,
 239                                      SmallVectorImpl<uint64_t> *Offsets) const {
 240   LLVMContext &Ctx = OrigArg.Ty->getContext();
 241
 242   SmallVector<EVT, 4> SplitVTs;
 243   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
 244
 245   if (SplitVTs.size() == 0)
 246     return;
 247
 248   if (SplitVTs.size() == 1) {
 249     // No splitting to do, but we want to replace the original type (e.g. [1 x
 250     // double] -> double).
 251     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
 252                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
 253                            OrigArg.IsFixed, OrigArg.OrigValue);
 254     return;
 255   }
 256
 257   // Create one ArgInfo for each virtual register in the original ArgInfo.
 258   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
 259
 260   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
 261       OrigArg.Ty, CallConv, false, DL);
 262   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
 263     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
 264     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
 265                            OrigArg.Flags[0], OrigArg.IsFixed);
 266     if (NeedsRegBlock)
 267       SplitArgs.back().Flags[0].setInConsecutiveRegs();
 268   }
 269
 270   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
 271 }
 272
 273 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
 274 static MachineInstrBuilder
 275 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 276                             ArrayRef<Register> SrcRegs) {
 277   MachineRegisterInfo &MRI = *B.getMRI();
 278   LLT LLTy = MRI.getType(DstRegs[0]);
 279   LLT PartLLT = MRI.getType(SrcRegs[0]);
 280
 281   // Deal with v3s16 split into v2s16
 282   LLT LCMTy = getCoverTy(LLTy, PartLLT);
 283   if (LCMTy == LLTy) {
 284     // Common case where no padding is needed.
 285     assert(DstRegs.size() == 1);
 286     return B.buildConcatVectors(DstRegs[0], SrcRegs);
 287   }
 288
 289   // We need to create an unmerge to the result registers, which may require
 290   // widening the original value.
 291   Register UnmergeSrcReg;
 292   if (LCMTy != PartLLT) {
 293     assert(DstRegs.size() == 1);
 294     return B.buildDeleteTrailingVectorElements(DstRegs[0],
 295                                                B.buildMerge(LCMTy, SrcRegs));
 296   } else {
 297     // We don't need to widen anything if we're extracting a scalar which was
 298     // promoted to a vector e.g. s8 -> v4s8 -> s8
 299     assert(SrcRegs.size() == 1);
 300     UnmergeSrcReg = SrcRegs[0];
 301   }
 302
 303   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
 304
 305   SmallVector<Register, 8> PadDstRegs(NumDst);
 306   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
 307
 308   // Create the excess dead defs for the unmerge.
 309   for (int I = DstRegs.size(); I != NumDst; ++I)
 310     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
 311
 312   if (PadDstRegs.size() == 1)
 313     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
 314   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
 315 }
 316
 317 /// Create a sequence of instructions to combine pieces split into register
 318 /// typed values to the original IR value. \p OrigRegs contains the destination
 319 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
 320 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
 321 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
 322                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
 323                               const ISD::ArgFlagsTy Flags) {
 324   MachineRegisterInfo &MRI = *B.getMRI();
 325
 326   if (PartLLT == LLTy) {
 327     // We should have avoided introducing a new virtual register, and just
 328     // directly assigned here.
 329     assert(OrigRegs[0] == Regs[0]);
 330     return;
 331   }
 332
 333   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
 334       Regs.size() == 1) {
 335     B.buildBitcast(OrigRegs[0], Regs[0]);
 336     return;
 337   }
 338
 339   // A vector PartLLT needs extending to LLTy's element size.
 340   // E.g. <2 x s64> = G_SEXT <2 x s32>.
 341   if (PartLLT.isVector() == LLTy.isVector() &&
 342       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
 343       (!PartLLT.isVector() ||
 344        PartLLT.getNumElements() == LLTy.getNumElements()) &&
 345       OrigRegs.size() == 1 && Regs.size() == 1) {
 346     Register SrcReg = Regs[0];
 347
 348     LLT LocTy = MRI.getType(SrcReg);
 349
 350     if (Flags.isSExt()) {
 351       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
 352                    .getReg(0);
 353     } else if (Flags.isZExt()) {
 354       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
 355                    .getReg(0);
 356     }
 357
 358     // Sometimes pointers are passed zero extended.
 359     LLT OrigTy = MRI.getType(OrigRegs[0]);
 360     if (OrigTy.isPointer()) {
 361       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
 362       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
 363       return;
 364     }
 365
 366     B.buildTrunc(OrigRegs[0], SrcReg);
 367     return;
 368   }
 369
 370   if (!LLTy.isVector() && !PartLLT.isVector()) {
 371     assert(OrigRegs.size() == 1);
 372     LLT OrigTy = MRI.getType(OrigRegs[0]);
 373
 374     unsigned SrcSize = PartLLT.getSizeInBits().getFixedSize() * Regs.size();
 375     if (SrcSize == OrigTy.getSizeInBits())
 376       B.buildMerge(OrigRegs[0], Regs);
 377     else {
 378       auto Widened = B.buildMerge(LLT::scalar(SrcSize), Regs);
 379       B.buildTrunc(OrigRegs[0], Widened);
 380     }
 381
 382     return;
 383   }
 384
 385   if (PartLLT.isVector()) {
 386     assert(OrigRegs.size() == 1);
 387     SmallVector<Register> CastRegs(Regs.begin(), Regs.end());
 388
 389     // If PartLLT is a mismatched vector in both number of elements and element
 390     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
 391     // have the same elt type, i.e. v4s32.
 392     if (PartLLT.getSizeInBits() > LLTy.getSizeInBits() &&
 393         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
 394         Regs.size() == 1) {
 395       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
 396                       .changeElementCount(PartLLT.getElementCount() * 2);
 397       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
 398       PartLLT = NewTy;
 399     }
 400
 401     if (LLTy.getScalarType() == PartLLT.getElementType()) {
 402       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
 403     } else {
 404       unsigned I = 0;
 405       LLT GCDTy = getGCDType(LLTy, PartLLT);
 406
 407       // We are both splitting a vector, and bitcasting its element types. Cast
 408       // the source pieces into the appropriate number of pieces with the result
 409       // element type.
 410       for (Register SrcReg : CastRegs)
 411         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
 412       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
 413     }
 414
 415     return;
 416   }
 417
 418   assert(LLTy.isVector() && !PartLLT.isVector());
 419
 420   LLT DstEltTy = LLTy.getElementType();
 421
 422   // Pointer information was discarded. We'll need to coerce some register types
 423   // to avoid violating type constraints.
 424   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
 425
 426   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
 427
 428   if (DstEltTy == PartLLT) {
 429     // Vector was trivially scalarized.
 430
 431     if (RealDstEltTy.isPointer()) {
 432       for (Register Reg : Regs)
 433         MRI.setType(Reg, RealDstEltTy);
 434     }
 435
 436     B.buildBuildVector(OrigRegs[0], Regs);
 437   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
 438     // Deal with vector with 64-bit elements decomposed to 32-bit
 439     // registers. Need to create intermediate 64-bit elements.
 440     SmallVector<Register, 8> EltMerges;
 441     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
 442
 443     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
 444
 445     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
 446       auto Merge = B.buildMerge(RealDstEltTy, Regs.take_front(PartsPerElt));
 447       // Fix the type in case this is really a vector of pointers.
 448       MRI.setType(Merge.getReg(0), RealDstEltTy);
 449       EltMerges.push_back(Merge.getReg(0));
 450       Regs = Regs.drop_front(PartsPerElt);
 451     }
 452
 453     B.buildBuildVector(OrigRegs[0], EltMerges);
 454   } else {
 455     // Vector was split, and elements promoted to a wider type.
 456     // FIXME: Should handle floating point promotions.
 457     LLT BVType = LLT::fixed_vector(LLTy.getNumElements(), PartLLT);
 458     auto BV = B.buildBuildVector(BVType, Regs);
 459     B.buildTrunc(OrigRegs[0], BV);
 460   }
 461 }
 462
 463 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
 464 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
 465 /// contain the type of scalar value extension if necessary.
 466 ///
 467 /// This is used for outgoing values (vregs to physregs)
 468 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 469                             Register SrcReg, LLT SrcTy, LLT PartTy,
 470                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
 471   // We could just insert a regular copy, but this is unreachable at the moment.
 472   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
 473
 474   const unsigned PartSize = PartTy.getSizeInBits();
 475
 476   if (PartTy.isVector() == SrcTy.isVector() &&
 477       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
 478     assert(DstRegs.size() == 1);
 479     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
 480     return;
 481   }
 482
 483   if (SrcTy.isVector() && !PartTy.isVector() &&
 484       PartSize > SrcTy.getElementType().getSizeInBits()) {
 485     // Vector was scalarized, and the elements extended.
 486     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
 487     for (int i = 0, e = DstRegs.size(); i != e; ++i)
 488       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
 489     return;
 490   }
 491
 492   LLT GCDTy = getGCDType(SrcTy, PartTy);
 493   if (GCDTy == PartTy) {
 494     // If this already evenly divisible, we can create a simple unmerge.
 495     B.buildUnmerge(DstRegs, SrcReg);
 496     return;
 497   }
 498
 499   MachineRegisterInfo &MRI = *B.getMRI();
 500   LLT DstTy = MRI.getType(DstRegs[0]);
 501   LLT LCMTy = getCoverTy(SrcTy, PartTy);
 502
 503   const unsigned DstSize = DstTy.getSizeInBits();
 504   const unsigned SrcSize = SrcTy.getSizeInBits();
 505   unsigned CoveringSize = LCMTy.getSizeInBits();
 506
 507   Register UnmergeSrc = SrcReg;
 508
 509   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
 510     // For scalars, it's common to be able to use a simple extension.
 511     if (SrcTy.isScalar() && DstTy.isScalar()) {
 512       CoveringSize = alignTo(SrcSize, DstSize);
 513       LLT CoverTy = LLT::scalar(CoveringSize);
 514       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
 515     } else {
 516       // Widen to the common type.
 517       // FIXME: This should respect the extend type
 518       Register Undef = B.buildUndef(SrcTy).getReg(0);
 519       SmallVector<Register, 8> MergeParts(1, SrcReg);
 520       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
 521         MergeParts.push_back(Undef);
 522       UnmergeSrc = B.buildMerge(LCMTy, MergeParts).getReg(0);
 523     }
 524   }
 525
 526   if (LCMTy.isVector() && CoveringSize != SrcSize)
 527     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
 528
 529   B.buildUnmerge(DstRegs, UnmergeSrc);
 530 }
 531
 532 bool CallLowering::determineAndHandleAssignments(
 533     ValueHandler &Handler, ValueAssigner &Assigner,
 534     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
 535     CallingConv::ID CallConv, bool IsVarArg,
 536     ArrayRef<Register> ThisReturnRegs) const {
 537   MachineFunction &MF = MIRBuilder.getMF();
 538   const Function &F = MF.getFunction();
 539   SmallVector<CCValAssign, 16> ArgLocs;
 540
 541   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
 542   if (!determineAssignments(Assigner, Args, CCInfo))
 543     return false;
 544
 545   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
 546                            ThisReturnRegs);
 547 }
 548
 549 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
 550   if (Flags.isSExt())
 551     return TargetOpcode::G_SEXT;
 552   if (Flags.isZExt())
 553     return TargetOpcode::G_ZEXT;
 554   return TargetOpcode::G_ANYEXT;
 555 }
 556
 557 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
 558                                         SmallVectorImpl<ArgInfo> &Args,
 559                                         CCState &CCInfo) const {
 560   LLVMContext &Ctx = CCInfo.getContext();
 561   const CallingConv::ID CallConv = CCInfo.getCallingConv();
 562
 563   unsigned NumArgs = Args.size();
 564   for (unsigned i = 0; i != NumArgs; ++i) {
 565     EVT CurVT = EVT::getEVT(Args[i].Ty);
 566
 567     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
 568
 569     // If we need to split the type over multiple regs, check it's a scenario
 570     // we currently support.
 571     unsigned NumParts =
 572         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
 573
 574     if (NumParts == 1) {
 575       // Try to use the register type if we couldn't assign the VT.
 576       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
 577                              Args[i].Flags[0], CCInfo))
 578         return false;
 579       continue;
 580     }
 581
 582     // For incoming arguments (physregs to vregs), we could have values in
 583     // physregs (or memlocs) which we want to extract and copy to vregs.
 584     // During this, we might have to deal with the LLT being split across
 585     // multiple regs, so we have to record this information for later.
 586     //
 587     // If we have outgoing args, then we have the opposite case. We have a
 588     // vreg with an LLT which we want to assign to a physical location, and
 589     // we might have to record that the value has to be split later.
 590
 591     // We're handling an incoming arg which is split over multiple regs.
 592     // E.g. passing an s128 on AArch64.
 593     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
 594     Args[i].Flags.clear();
 595
 596     for (unsigned Part = 0; Part < NumParts; ++Part) {
 597       ISD::ArgFlagsTy Flags = OrigFlags;
 598       if (Part == 0) {
 599         Flags.setSplit();
 600       } else {
 601         Flags.setOrigAlign(Align(1));
 602         if (Part == NumParts - 1)
 603           Flags.setSplitEnd();
 604       }
 605
 606       Args[i].Flags.push_back(Flags);
 607       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
 608                              Args[i].Flags[Part], CCInfo)) {
 609         // Still couldn't assign this smaller part type for some reason.
 610         return false;
 611       }
 612     }
 613   }
 614
 615   return true;
 616 }
 617
 618 bool CallLowering::handleAssignments(ValueHandler &Handler,
 619                                      SmallVectorImpl<ArgInfo> &Args,
 620                                      CCState &CCInfo,
 621                                      SmallVectorImpl<CCValAssign> &ArgLocs,
 622                                      MachineIRBuilder &MIRBuilder,
 623                                      ArrayRef<Register> ThisReturnRegs) const {
 624   MachineFunction &MF = MIRBuilder.getMF();
 625   MachineRegisterInfo &MRI = MF.getRegInfo();
 626   const Function &F = MF.getFunction();
 627   const DataLayout &DL = F.getParent()->getDataLayout();
 628
 629   const unsigned NumArgs = Args.size();
 630
 631   // Stores thunks for outgoing register assignments. This is used so we delay
 632   // generating register copies until mem loc assignments are done. We do this
 633   // so that if the target is using the delayed stack protector feature, we can
 634   // find the split point of the block accurately. E.g. if we have:
 635   // G_STORE %val, %memloc
 636   // $x0 = COPY %foo
 637   // $x1 = COPY %bar
 638   // CALL func
 639   // ... then the split point for the block will correctly be at, and including,
 640   // the copy to $x0. If instead the G_STORE instruction immediately precedes
 641   // the CALL, then we'd prematurely choose the CALL as the split point, thus
 642   // generating a split block with a CALL that uses undefined physregs.
 643   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
 644
 645   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
 646     assert(j < ArgLocs.size() && "Skipped too many arg locs");
 647     CCValAssign &VA = ArgLocs[j];
 648     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
 649
 650     if (VA.needsCustom()) {
 651       std::function<void()> Thunk;
 652       unsigned NumArgRegs = Handler.assignCustomValue(
 653           Args[i], makeArrayRef(ArgLocs).slice(j), &Thunk);
 654       if (Thunk)
 655         DelayedOutgoingRegAssignments.emplace_back(Thunk);
 656       if (!NumArgRegs)
 657         return false;
 658       j += NumArgRegs;
 659       continue;
 660     }
 661
 662     const MVT ValVT = VA.getValVT();
 663     const MVT LocVT = VA.getLocVT();
 664
 665     const LLT LocTy(LocVT);
 666     const LLT ValTy(ValVT);
 667     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
 668     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
 669     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
 670
 671     // Expected to be multiple regs for a single incoming arg.
 672     // There should be Regs.size() ArgLocs per argument.
 673     // This should be the same as getNumRegistersForCallingConv
 674     const unsigned NumParts = Args[i].Flags.size();
 675
 676     // Now split the registers into the assigned types.
 677     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
 678
 679     if (NumParts != 1 || NewLLT != OrigTy) {
 680       // If we can't directly assign the register, we need one or more
 681       // intermediate values.
 682       Args[i].Regs.resize(NumParts);
 683
 684       // For each split register, create and assign a vreg that will store
 685       // the incoming component of the larger value. These will later be
 686       // merged to form the final vreg.
 687       for (unsigned Part = 0; Part < NumParts; ++Part)
 688         Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
 689     }
 690
 691     assert((j + (NumParts - 1)) < ArgLocs.size() &&
 692            "Too many regs for number of args");
 693
 694     // Coerce into outgoing value types before register assignment.
 695     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy) {
 696       assert(Args[i].OrigRegs.size() == 1);
 697       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
 698                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
 699     }
 700
 701     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
 702     for (unsigned Part = 0; Part < NumParts; ++Part) {
 703       Register ArgReg = Args[i].Regs[Part];
 704       // There should be Regs.size() ArgLocs per argument.
 705       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
 706       CCValAssign &VA = ArgLocs[j + Idx];
 707       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
 708
 709       if (VA.isMemLoc() && !Flags.isByVal()) {
 710         // Individual pieces may have been spilled to the stack and others
 711         // passed in registers.
 712
 713         // TODO: The memory size may be larger than the value we need to
 714         // store. We may need to adjust the offset for big endian targets.
 715         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
 716
 717         MachinePointerInfo MPO;
 718         Register StackAddr = Handler.getStackAddress(
 719             MemTy.getSizeInBytes(), VA.getLocMemOffset(), MPO, Flags);
 720
 721         Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO, VA);
 722         continue;
 723       }
 724
 725       if (VA.isMemLoc() && Flags.isByVal()) {
 726         assert(Args[i].Regs.size() == 1 &&
 727                "didn't expect split byval pointer");
 728
 729         if (Handler.isIncomingArgumentHandler()) {
 730           // We just need to copy the frame index value to the pointer.
 731           MachinePointerInfo MPO;
 732           Register StackAddr = Handler.getStackAddress(
 733               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
 734           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
 735         } else {
 736           // For outgoing byval arguments, insert the implicit copy byval
 737           // implies, such that writes in the callee do not modify the caller's
 738           // value.
 739           uint64_t MemSize = Flags.getByValSize();
 740           int64_t Offset = VA.getLocMemOffset();
 741
 742           MachinePointerInfo DstMPO;
 743           Register StackAddr =
 744               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
 745
 746           MachinePointerInfo SrcMPO(Args[i].OrigValue);
 747           if (!Args[i].OrigValue) {
 748             // We still need to accurately track the stack address space if we
 749             // don't know the underlying value.
 750             const LLT PtrTy = MRI.getType(StackAddr);
 751             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
 752           }
 753
 754           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
 755                                     inferAlignFromPtrInfo(MF, DstMPO));
 756
 757           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
 758                                     inferAlignFromPtrInfo(MF, SrcMPO));
 759
 760           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
 761                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
 762                                      MemSize, VA);
 763         }
 764         continue;
 765       }
 766
 767       assert(!VA.needsCustom() && "custom loc should have been handled already");
 768
 769       if (i == 0 && !ThisReturnRegs.empty() &&
 770           Handler.isIncomingArgumentHandler() &&
 771           isTypeIsValidForThisReturn(ValVT)) {
 772         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
 773         continue;
 774       }
 775
 776       if (Handler.isIncomingArgumentHandler())
 777         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
 778       else {
 779         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
 780           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
 781         });
 782       }
 783     }
 784
 785     // Now that all pieces have been assigned, re-pack the register typed values
 786     // into the original value typed registers.
 787     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT) {
 788       // Merge the split registers into the expected larger result vregs of
 789       // the original call.
 790       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
 791                         LocTy, Args[i].Flags[0]);
 792     }
 793
 794     j += NumParts - 1;
 795   }
 796   for (auto &Fn : DelayedOutgoingRegAssignments)
 797     Fn();
 798
 799   return true;
 800 }
 801
 802 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
 803                                    ArrayRef<Register> VRegs, Register DemoteReg,
 804                                    int FI) const {
 805   MachineFunction &MF = MIRBuilder.getMF();
 806   MachineRegisterInfo &MRI = MF.getRegInfo();
 807   const DataLayout &DL = MF.getDataLayout();
 808
 809   SmallVector<EVT, 4> SplitVTs;
 810   SmallVector<uint64_t, 4> Offsets;
 811   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
 812
 813   assert(VRegs.size() == SplitVTs.size());
 814
 815   unsigned NumValues = SplitVTs.size();
 816   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
 817   Type *RetPtrTy = RetTy->getPointerTo(DL.getAllocaAddrSpace());
 818   LLT OffsetLLTy = getLLTForType(*DL.getIntPtrType(RetPtrTy), DL);
 819
 820   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
 821
 822   for (unsigned I = 0; I < NumValues; ++I) {
 823     Register Addr;
 824     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
 825     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
 826                                         MRI.getType(VRegs[I]),
 827                                         commonAlignment(BaseAlign, Offsets[I]));
 828     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
 829   }
 830 }
 831
 832 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
 833                                     ArrayRef<Register> VRegs,
 834                                     Register DemoteReg) const {
 835   MachineFunction &MF = MIRBuilder.getMF();
 836   MachineRegisterInfo &MRI = MF.getRegInfo();
 837   const DataLayout &DL = MF.getDataLayout();
 838
 839   SmallVector<EVT, 4> SplitVTs;
 840   SmallVector<uint64_t, 4> Offsets;
 841   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
 842
 843   assert(VRegs.size() == SplitVTs.size());
 844
 845   unsigned NumValues = SplitVTs.size();
 846   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
 847   unsigned AS = DL.getAllocaAddrSpace();
 848   LLT OffsetLLTy =
 849       getLLTForType(*DL.getIntPtrType(RetTy->getPointerTo(AS)), DL);
 850
 851   MachinePointerInfo PtrInfo(AS);
 852
 853   for (unsigned I = 0; I < NumValues; ++I) {
 854     Register Addr;
 855     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
 856     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
 857                                         MRI.getType(VRegs[I]),
 858                                         commonAlignment(BaseAlign, Offsets[I]));
 859     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
 860   }
 861 }
 862
 863 void CallLowering::insertSRetIncomingArgument(
 864     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
 865     MachineRegisterInfo &MRI, const DataLayout &DL) const {
 866   unsigned AS = DL.getAllocaAddrSpace();
 867   DemoteReg = MRI.createGenericVirtualRegister(
 868       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
 869
 870   Type *PtrTy = PointerType::get(F.getReturnType(), AS);
 871
 872   SmallVector<EVT, 1> ValueVTs;
 873   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
 874
 875   // NOTE: Assume that a pointer won't get split into more than one VT.
 876   assert(ValueVTs.size() == 1);
 877
 878   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
 879                     ArgInfo::NoArgIndex);
 880   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
 881   DemoteArg.Flags[0].setSRet();
 882   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
 883 }
 884
 885 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
 886                                               const CallBase &CB,
 887                                               CallLoweringInfo &Info) const {
 888   const DataLayout &DL = MIRBuilder.getDataLayout();
 889   Type *RetTy = CB.getType();
 890   unsigned AS = DL.getAllocaAddrSpace();
 891   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
 892
 893   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
 894       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
 895
 896   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
 897   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy, AS),
 898                     ArgInfo::NoArgIndex);
 899   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
 900   DemoteArg.Flags[0].setSRet();
 901
 902   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
 903   Info.DemoteStackIndex = FI;
 904   Info.DemoteRegister = DemoteReg;
 905 }
 906
 907 bool CallLowering::checkReturn(CCState &CCInfo,
 908                                SmallVectorImpl<BaseArgInfo> &Outs,
 909                                CCAssignFn *Fn) const {
 910   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
 911     MVT VT = MVT::getVT(Outs[I].Ty);
 912     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
 913       return false;
 914   }
 915   return true;
 916 }
 917
 918 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
 919                                  AttributeList Attrs,
 920                                  SmallVectorImpl<BaseArgInfo> &Outs,
 921                                  const DataLayout &DL) const {
 922   LLVMContext &Context = RetTy->getContext();
 923   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
 924
 925   SmallVector<EVT, 4> SplitVTs;
 926   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
 927   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
 928
 929   for (EVT VT : SplitVTs) {
 930     unsigned NumParts =
 931         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
 932     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
 933     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
 934
 935     for (unsigned I = 0; I < NumParts; ++I) {
 936       Outs.emplace_back(PartTy, Flags);
 937     }
 938   }
 939 }
 940
 941 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
 942   const auto &F = MF.getFunction();
 943   Type *ReturnType = F.getReturnType();
 944   CallingConv::ID CallConv = F.getCallingConv();
 945
 946   SmallVector<BaseArgInfo, 4> SplitArgs;
 947   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
 948                 MF.getDataLayout());
 949   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
 950 }
 951
 952 bool CallLowering::parametersInCSRMatch(
 953     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
 954     const SmallVectorImpl<CCValAssign> &OutLocs,
 955     const SmallVectorImpl<ArgInfo> &OutArgs) const {
 956   for (unsigned i = 0; i < OutLocs.size(); ++i) {
 957     auto &ArgLoc = OutLocs[i];
 958     // If it's not a register, it's fine.
 959     if (!ArgLoc.isRegLoc())
 960       continue;
 961
 962     MCRegister PhysReg = ArgLoc.getLocReg();
 963
 964     // Only look at callee-saved registers.
 965     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
 966       continue;
 967
 968     LLVM_DEBUG(
 969         dbgs()
 970         << "... Call has an argument passed in a callee-saved register.\n");
 971
 972     // Check if it was copied from.
 973     const ArgInfo &OutInfo = OutArgs[i];
 974
 975     if (OutInfo.Regs.size() > 1) {
 976       LLVM_DEBUG(
 977           dbgs() << "... Cannot handle arguments in multiple registers.\n");
 978       return false;
 979     }
 980
 981     // Check if we copy the register, walking through copies from virtual
 982     // registers. Note that getDefIgnoringCopies does not ignore copies from
 983     // physical registers.
 984     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
 985     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
 986       LLVM_DEBUG(
 987           dbgs()
 988           << "... Parameter was not copied into a VReg, cannot tail call.\n");
 989       return false;
 990     }
 991
 992     // Got a copy. Verify that it's the same as the register we want.
 993     Register CopyRHS = RegDef->getOperand(1).getReg();
 994     if (CopyRHS != PhysReg) {
 995       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
 996                            "VReg, cannot tail call.\n");
 997       return false;
 998     }
 999   }
1000
1001   return true;
1002 }
1003
1004 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1005                                      MachineFunction &MF,
1006                                      SmallVectorImpl<ArgInfo> &InArgs,
1007                                      ValueAssigner &CalleeAssigner,
1008                                      ValueAssigner &CallerAssigner) const {
1009   const Function &F = MF.getFunction();
1010   CallingConv::ID CalleeCC = Info.CallConv;
1011   CallingConv::ID CallerCC = F.getCallingConv();
1012
1013   if (CallerCC == CalleeCC)
1014     return true;
1015
1016   SmallVector<CCValAssign, 16> ArgLocs1;
1017   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1018   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1019     return false;
1020
1021   SmallVector<CCValAssign, 16> ArgLocs2;
1022   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1023   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1024     return false;
1025
1026   // We need the argument locations to match up exactly. If there's more in
1027   // one than the other, then we are done.
1028   if (ArgLocs1.size() != ArgLocs2.size())
1029     return false;
1030
1031   // Make sure that each location is passed in exactly the same way.
1032   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1033     const CCValAssign &Loc1 = ArgLocs1[i];
1034     const CCValAssign &Loc2 = ArgLocs2[i];
1035
1036     // We need both of them to be the same. So if one is a register and one
1037     // isn't, we're done.
1038     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1039       return false;
1040
1041     if (Loc1.isRegLoc()) {
1042       // If they don't have the same register location, we're done.
1043       if (Loc1.getLocReg() != Loc2.getLocReg())
1044         return false;
1045
1046       // They matched, so we can move to the next ArgLoc.
1047       continue;
1048     }
1049
1050     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1051     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1052       return false;
1053   }
1054
1055   return true;
1056 }
1057
1058 LLT CallLowering::ValueHandler::getStackValueStoreType(
1059     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1060   const MVT ValVT = VA.getValVT();
1061   if (ValVT != MVT::iPTR) {
1062     LLT ValTy(ValVT);
1063
1064     // We lost the pointeriness going through CCValAssign, so try to restore it
1065     // based on the flags.
1066     if (Flags.isPointer()) {
1067       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1068                                ValTy.getScalarSizeInBits());
1069       if (ValVT.isVector())
1070         return LLT::vector(ValTy.getElementCount(), PtrTy);
1071       return PtrTy;
1072     }
1073
1074     return ValTy;
1075   }
1076
1077   unsigned AddrSpace = Flags.getPointerAddrSpace();
1078   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1079 }
1080
1081 void CallLowering::ValueHandler::copyArgumentMemory(
1082     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1083     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1084     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1085     CCValAssign &VA) const {
1086   MachineFunction &MF = MIRBuilder.getMF();
1087   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1088       SrcPtrInfo,
1089       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1090       SrcAlign);
1091
1092   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1093       DstPtrInfo,
1094       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1095       MemSize, DstAlign);
1096
1097   const LLT PtrTy = MRI.getType(DstPtr);
1098   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1099
1100   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1101   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1102 }
1103
1104 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1105                                                     CCValAssign &VA,
1106                                                     unsigned MaxSizeBits) {
1107   LLT LocTy{VA.getLocVT()};
1108   LLT ValTy{VA.getValVT()};
1109
1110   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1111     return ValReg;
1112
1113   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1114     if (MaxSizeBits <= ValTy.getSizeInBits())
1115       return ValReg;
1116     LocTy = LLT::scalar(MaxSizeBits);
1117   }
1118
1119   const LLT ValRegTy = MRI.getType(ValReg);
1120   if (ValRegTy.isPointer()) {
1121     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1122     // we have to cast to do the extension.
1123     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1124     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1125   }
1126
1127   switch (VA.getLocInfo()) {
1128   default: break;
1129   case CCValAssign::Full:
1130   case CCValAssign::BCvt:
1131     // FIXME: bitconverting between vector types may or may not be a
1132     // nop in big-endian situations.
1133     return ValReg;
1134   case CCValAssign::AExt: {
1135     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1136     return MIB.getReg(0);
1137   }
1138   case CCValAssign::SExt: {
1139     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1140     MIRBuilder.buildSExt(NewReg, ValReg);
1141     return NewReg;
1142   }
1143   case CCValAssign::ZExt: {
1144     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1145     MIRBuilder.buildZExt(NewReg, ValReg);
1146     return NewReg;
1147   }
1148   }
1149   llvm_unreachable("unable to extend register");
1150 }
1151
1152 void CallLowering::ValueAssigner::anchor() {}
1153
1154 Register CallLowering::IncomingValueHandler::buildExtensionHint(CCValAssign &VA,
1155                                                                 Register SrcReg,
1156                                                                 LLT NarrowTy) {
1157   switch (VA.getLocInfo()) {
1158   case CCValAssign::LocInfo::ZExt: {
1159     return MIRBuilder
1160         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1161                          NarrowTy.getScalarSizeInBits())
1162         .getReg(0);
1163   }
1164   case CCValAssign::LocInfo::SExt: {
1165     return MIRBuilder
1166         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1167                          NarrowTy.getScalarSizeInBits())
1168         .getReg(0);
1169     break;
1170   }
1171   default:
1172     return SrcReg;
1173   }
1174 }
1175
1176 /// Check if we can use a basic COPY instruction between the two types.
1177 ///
1178 /// We're currently building on top of the infrastructure using MVT, which loses
1179 /// pointer information in the CCValAssign. We accept copies from physical
1180 /// registers that have been reported as integers if it's to an equivalent sized
1181 /// pointer LLT.
1182 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1183   if (SrcTy == DstTy)
1184     return true;
1185
1186   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1187     return false;
1188
1189   SrcTy = SrcTy.getScalarType();
1190   DstTy = DstTy.getScalarType();
1191
1192   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1193          (DstTy.isScalar() && SrcTy.isPointer());
1194 }
1195
1196 void CallLowering::IncomingValueHandler::assignValueToReg(Register ValVReg,
1197                                                           Register PhysReg,
1198                                                           CCValAssign VA) {
1199   const MVT LocVT = VA.getLocVT();
1200   const LLT LocTy(LocVT);
1201   const LLT RegTy = MRI.getType(ValVReg);
1202
1203   if (isCopyCompatibleType(RegTy, LocTy)) {
1204     MIRBuilder.buildCopy(ValVReg, PhysReg);
1205     return;
1206   }
1207
1208   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1209   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1210   MIRBuilder.buildTrunc(ValVReg, Hint);
1211 }