llvm/lib/CodeGen/GlobalISel/CallLowering.cpp

   1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements some simple delegations needed for call lowering.
  11 ///
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
  15 #include "llvm/CodeGen/Analysis.h"
  16 #include "llvm/CodeGen/CallingConvLower.h"
  17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  18 #include "llvm/CodeGen/GlobalISel/Utils.h"
  19 #include "llvm/CodeGen/MachineFrameInfo.h"
  20 #include "llvm/CodeGen/MachineOperand.h"
  21 #include "llvm/CodeGen/MachineRegisterInfo.h"
  22 #include "llvm/CodeGen/TargetLowering.h"
  23 #include "llvm/IR/DataLayout.h"
  24 #include "llvm/IR/LLVMContext.h"
  25 #include "llvm/IR/Module.h"
  26 #include "llvm/Target/TargetMachine.h"
  27
  28 #define DEBUG_TYPE "call-lowering"
  29
  30 using namespace llvm;
  31
  32 void CallLowering::anchor() {}
  33
  34 /// Helper function which updates \p Flags when \p AttrFn returns true.
  35 static void
  36 addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,
  37                     const std::function<bool(Attribute::AttrKind)> &AttrFn) {
  38   // TODO: There are missing flags. Add them here.
  39   if (AttrFn(Attribute::SExt))
  40     Flags.setSExt();
  41   if (AttrFn(Attribute::ZExt))
  42     Flags.setZExt();
  43   if (AttrFn(Attribute::InReg))
  44     Flags.setInReg();
  45   if (AttrFn(Attribute::StructRet))
  46     Flags.setSRet();
  47   if (AttrFn(Attribute::Nest))
  48     Flags.setNest();
  49   if (AttrFn(Attribute::ByVal))
  50     Flags.setByVal();
  51   if (AttrFn(Attribute::ByRef))
  52     Flags.setByRef();
  53   if (AttrFn(Attribute::Preallocated))
  54     Flags.setPreallocated();
  55   if (AttrFn(Attribute::InAlloca))
  56     Flags.setInAlloca();
  57   if (AttrFn(Attribute::Returned))
  58     Flags.setReturned();
  59   if (AttrFn(Attribute::SwiftSelf))
  60     Flags.setSwiftSelf();
  61   if (AttrFn(Attribute::SwiftAsync))
  62     Flags.setSwiftAsync();
  63   if (AttrFn(Attribute::SwiftError))
  64     Flags.setSwiftError();
  65 }
  66
  67 ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,
  68                                                      unsigned ArgIdx) const {
  69   ISD::ArgFlagsTy Flags;
  70   addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {
  71     return Call.paramHasAttr(ArgIdx, Attr);
  72   });
  73   return Flags;
  74 }
  75
  76 ISD::ArgFlagsTy
  77 CallLowering::getAttributesForReturn(const CallBase &Call) const {
  78   ISD::ArgFlagsTy Flags;
  79   addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {
  80     return Call.hasRetAttr(Attr);
  81   });
  82   return Flags;
  83 }
  84
  85 void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,
  86                                              const AttributeList &Attrs,
  87                                              unsigned OpIdx) const {
  88   addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {
  89     return Attrs.hasAttributeAtIndex(OpIdx, Attr);
  90   });
  91 }
  92
  93 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
  94                              ArrayRef<Register> ResRegs,
  95                              ArrayRef<ArrayRef<Register>> ArgRegs,
  96                              Register SwiftErrorVReg,
  97                              std::optional<PtrAuthInfo> PAI,
  98                              Register ConvergenceCtrlToken,
  99                              std::function<unsigned()> GetCalleeReg) const {
 100   CallLoweringInfo Info;
 101   const DataLayout &DL = MIRBuilder.getDataLayout();
 102   MachineFunction &MF = MIRBuilder.getMF();
 103   MachineRegisterInfo &MRI = MF.getRegInfo();
 104   bool CanBeTailCalled = CB.isTailCall() &&
 105                          isInTailCallPosition(CB, MF.getTarget()) &&
 106                          (MF.getFunction()
 107                               .getFnAttribute("disable-tail-calls")
 108                               .getValueAsString() != "true");
 109
 110   CallingConv::ID CallConv = CB.getCallingConv();
 111   Type *RetTy = CB.getType();
 112   bool IsVarArg = CB.getFunctionType()->isVarArg();
 113
 114   SmallVector<BaseArgInfo, 4> SplitArgs;
 115   getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);
 116   Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);
 117
 118   Info.IsConvergent = CB.isConvergent();
 119
 120   if (!Info.CanLowerReturn) {
 121     // Callee requires sret demotion.
 122     insertSRetOutgoingArgument(MIRBuilder, CB, Info);
 123
 124     // The sret demotion isn't compatible with tail-calls, since the sret
 125     // argument points into the caller's stack frame.
 126     CanBeTailCalled = false;
 127   }
 128
 129   // First step is to marshall all the function's parameters into the correct
 130   // physregs and memory locations. Gather the sequence of argument types that
 131   // we'll pass to the assigner function.
 132   unsigned i = 0;
 133   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
 134   for (const auto &Arg : CB.args()) {
 135     ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
 136                     i < NumFixedArgs};
 137     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
 138
 139     // If we have an explicit sret argument that is an Instruction, (i.e., it
 140     // might point to function-local memory), we can't meaningfully tail-call.
 141     if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))
 142       CanBeTailCalled = false;
 143
 144     Info.OrigArgs.push_back(OrigArg);
 145     ++i;
 146   }
 147
 148   // Try looking through a bitcast from one function type to another.
 149   // Commonly happens with calls to objc_msgSend().
 150   const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();
 151
 152   // If IRTranslator chose to drop the ptrauth info, we can turn this into
 153   // a direct call.
 154   if (!PAI && CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
 155     CalleeV = cast<ConstantPtrAuth>(CalleeV)->getPointer();
 156     assert(isa<Function>(CalleeV));
 157   }
 158
 159   if (const Function *F = dyn_cast<Function>(CalleeV)) {
 160     if (F->hasFnAttribute(Attribute::NonLazyBind)) {
 161       LLT Ty = getLLTForType(*F->getType(), DL);
 162       Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0);
 163       Info.Callee = MachineOperand::CreateReg(Reg, false);
 164     } else {
 165       Info.Callee = MachineOperand::CreateGA(F, 0);
 166     }
 167   } else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) {
 168     // IR IFuncs and Aliases can't be forward declared (only defined), so the
 169     // callee must be in the same TU and therefore we can direct-call it without
 170     // worrying about it being out of range.
 171     Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0);
 172   } else
 173     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
 174
 175   Register ReturnHintAlignReg;
 176   Align ReturnHintAlign;
 177
 178   Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};
 179
 180   if (!Info.OrigRet.Ty->isVoidTy()) {
 181     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
 182
 183     if (MaybeAlign Alignment = CB.getRetAlign()) {
 184       if (*Alignment > Align(1)) {
 185         ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);
 186         Info.OrigRet.Regs[0] = ReturnHintAlignReg;
 187         ReturnHintAlign = *Alignment;
 188       }
 189     }
 190   }
 191
 192   auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);
 193   if (Bundle && CB.isIndirectCall()) {
 194     Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
 195     assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
 196   }
 197
 198   Info.CB = &CB;
 199   Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
 200   Info.CallConv = CallConv;
 201   Info.SwiftErrorVReg = SwiftErrorVReg;
 202   Info.PAI = PAI;
 203   Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
 204   Info.IsMustTailCall = CB.isMustTailCall();
 205   Info.IsTailCall = CanBeTailCalled;
 206   Info.IsVarArg = IsVarArg;
 207   if (!lowerCall(MIRBuilder, Info))
 208     return false;
 209
 210   if (ReturnHintAlignReg && !Info.LoweredTailCall) {
 211     MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,
 212                                 ReturnHintAlign);
 213   }
 214
 215   return true;
 216 }
 217
 218 template <typename FuncInfoTy>
 219 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 220                                const DataLayout &DL,
 221                                const FuncInfoTy &FuncInfo) const {
 222   auto &Flags = Arg.Flags[0];
 223   const AttributeList &Attrs = FuncInfo.getAttributes();
 224   addArgFlagsFromAttributes(Flags, Attrs, OpIdx);
 225
 226   PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());
 227   if (PtrTy) {
 228     Flags.setPointer();
 229     Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());
 230   }
 231
 232   Align MemAlign = DL.getABITypeAlign(Arg.Ty);
 233   if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
 234       Flags.isByRef()) {
 235     assert(OpIdx >= AttributeList::FirstArgIndex);
 236     unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;
 237
 238     Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);
 239     if (!ElementTy)
 240       ElementTy = FuncInfo.getParamByRefType(ParamIdx);
 241     if (!ElementTy)
 242       ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);
 243     if (!ElementTy)
 244       ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);
 245
 246     assert(ElementTy && "Must have byval, inalloca or preallocated type");
 247
 248     uint64_t MemSize = DL.getTypeAllocSize(ElementTy);
 249     if (Flags.isByRef())
 250       Flags.setByRefSize(MemSize);
 251     else
 252       Flags.setByValSize(MemSize);
 253
 254     // For ByVal, alignment should be passed from FE.  BE will guess if
 255     // this info is not there but there are cases it cannot get right.
 256     if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))
 257       MemAlign = *ParamAlign;
 258     else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))
 259       MemAlign = *ParamAlign;
 260     else
 261       MemAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
 262   } else if (OpIdx >= AttributeList::FirstArgIndex) {
 263     if (auto ParamAlign =
 264             FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))
 265       MemAlign = *ParamAlign;
 266   }
 267   Flags.setMemAlign(MemAlign);
 268   Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));
 269
 270   // Don't try to use the returned attribute if the argument is marked as
 271   // swiftself, since it won't be passed in x0.
 272   if (Flags.isSwiftSelf())
 273     Flags.setReturned(false);
 274 }
 275
 276 template void
 277 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 278                                     const DataLayout &DL,
 279                                     const Function &FuncInfo) const;
 280
 281 template void
 282 CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 283                                     const DataLayout &DL,
 284                                     const CallBase &FuncInfo) const;
 285
 286 void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
 287                                      SmallVectorImpl<ArgInfo> &SplitArgs,
 288                                      const DataLayout &DL,
 289                                      CallingConv::ID CallConv,
 290                                      SmallVectorImpl<uint64_t> *Offsets) const {
 291   LLVMContext &Ctx = OrigArg.Ty->getContext();
 292
 293   SmallVector<EVT, 4> SplitVTs;
 294   ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, Offsets, 0);
 295
 296   if (SplitVTs.size() == 0)
 297     return;
 298
 299   if (SplitVTs.size() == 1) {
 300     // No splitting to do, but we want to replace the original type (e.g. [1 x
 301     // double] -> double).
 302     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
 303                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
 304                            OrigArg.IsFixed, OrigArg.OrigValue);
 305     return;
 306   }
 307
 308   // Create one ArgInfo for each virtual register in the original ArgInfo.
 309   assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");
 310
 311   bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
 312       OrigArg.Ty, CallConv, false, DL);
 313   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
 314     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
 315     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
 316                            OrigArg.Flags[0], OrigArg.IsFixed);
 317     if (NeedsRegBlock)
 318       SplitArgs.back().Flags[0].setInConsecutiveRegs();
 319   }
 320
 321   SplitArgs.back().Flags[0].setInConsecutiveRegsLast();
 322 }
 323
 324 /// Pack values \p SrcRegs to cover the vector type result \p DstRegs.
 325 static MachineInstrBuilder
 326 mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 327                             ArrayRef<Register> SrcRegs) {
 328   MachineRegisterInfo &MRI = *B.getMRI();
 329   LLT LLTy = MRI.getType(DstRegs[0]);
 330   LLT PartLLT = MRI.getType(SrcRegs[0]);
 331
 332   // Deal with v3s16 split into v2s16
 333   LLT LCMTy = getCoverTy(LLTy, PartLLT);
 334   if (LCMTy == LLTy) {
 335     // Common case where no padding is needed.
 336     assert(DstRegs.size() == 1);
 337     return B.buildConcatVectors(DstRegs[0], SrcRegs);
 338   }
 339
 340   // We need to create an unmerge to the result registers, which may require
 341   // widening the original value.
 342   Register UnmergeSrcReg;
 343   if (LCMTy != PartLLT) {
 344     assert(DstRegs.size() == 1);
 345     return B.buildDeleteTrailingVectorElements(
 346         DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));
 347   } else {
 348     // We don't need to widen anything if we're extracting a scalar which was
 349     // promoted to a vector e.g. s8 -> v4s8 -> s8
 350     assert(SrcRegs.size() == 1);
 351     UnmergeSrcReg = SrcRegs[0];
 352   }
 353
 354   int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();
 355
 356   SmallVector<Register, 8> PadDstRegs(NumDst);
 357   std::copy(DstRegs.begin(), DstRegs.end(), PadDstRegs.begin());
 358
 359   // Create the excess dead defs for the unmerge.
 360   for (int I = DstRegs.size(); I != NumDst; ++I)
 361     PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);
 362
 363   if (PadDstRegs.size() == 1)
 364     return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);
 365   return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);
 366 }
 367
 368 /// Create a sequence of instructions to combine pieces split into register
 369 /// typed values to the original IR value. \p OrigRegs contains the destination
 370 /// value registers of type \p LLTy, and \p Regs contains the legalized pieces
 371 /// with type \p PartLLT. This is used for incoming values (physregs to vregs).
 372 static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,
 373                               ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,
 374                               const ISD::ArgFlagsTy Flags) {
 375   MachineRegisterInfo &MRI = *B.getMRI();
 376
 377   if (PartLLT == LLTy) {
 378     // We should have avoided introducing a new virtual register, and just
 379     // directly assigned here.
 380     assert(OrigRegs[0] == Regs[0]);
 381     return;
 382   }
 383
 384   if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&
 385       Regs.size() == 1) {
 386     B.buildBitcast(OrigRegs[0], Regs[0]);
 387     return;
 388   }
 389
 390   // A vector PartLLT needs extending to LLTy's element size.
 391   // E.g. <2 x s64> = G_SEXT <2 x s32>.
 392   if (PartLLT.isVector() == LLTy.isVector() &&
 393       PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&
 394       (!PartLLT.isVector() ||
 395        PartLLT.getElementCount() == LLTy.getElementCount()) &&
 396       OrigRegs.size() == 1 && Regs.size() == 1) {
 397     Register SrcReg = Regs[0];
 398
 399     LLT LocTy = MRI.getType(SrcReg);
 400
 401     if (Flags.isSExt()) {
 402       SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
 403                    .getReg(0);
 404     } else if (Flags.isZExt()) {
 405       SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())
 406                    .getReg(0);
 407     }
 408
 409     // Sometimes pointers are passed zero extended.
 410     LLT OrigTy = MRI.getType(OrigRegs[0]);
 411     if (OrigTy.isPointer()) {
 412       LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());
 413       B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));
 414       return;
 415     }
 416
 417     B.buildTrunc(OrigRegs[0], SrcReg);
 418     return;
 419   }
 420
 421   if (!LLTy.isVector() && !PartLLT.isVector()) {
 422     assert(OrigRegs.size() == 1);
 423     LLT OrigTy = MRI.getType(OrigRegs[0]);
 424
 425     unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();
 426     if (SrcSize == OrigTy.getSizeInBits())
 427       B.buildMergeValues(OrigRegs[0], Regs);
 428     else {
 429       auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);
 430       B.buildTrunc(OrigRegs[0], Widened);
 431     }
 432
 433     return;
 434   }
 435
 436   if (PartLLT.isVector()) {
 437     assert(OrigRegs.size() == 1);
 438     SmallVector<Register> CastRegs(Regs);
 439
 440     // If PartLLT is a mismatched vector in both number of elements and element
 441     // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to
 442     // have the same elt type, i.e. v4s32.
 443     // TODO: Extend this coersion to element multiples other than just 2.
 444     if (TypeSize::isKnownGT(PartLLT.getSizeInBits(), LLTy.getSizeInBits()) &&
 445         PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&
 446         Regs.size() == 1) {
 447       LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())
 448                       .changeElementCount(PartLLT.getElementCount() * 2);
 449       CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);
 450       PartLLT = NewTy;
 451     }
 452
 453     if (LLTy.getScalarType() == PartLLT.getElementType()) {
 454       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
 455     } else {
 456       unsigned I = 0;
 457       LLT GCDTy = getGCDType(LLTy, PartLLT);
 458
 459       // We are both splitting a vector, and bitcasting its element types. Cast
 460       // the source pieces into the appropriate number of pieces with the result
 461       // element type.
 462       for (Register SrcReg : CastRegs)
 463         CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);
 464       mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);
 465     }
 466
 467     return;
 468   }
 469
 470   assert(LLTy.isVector() && !PartLLT.isVector());
 471
 472   LLT DstEltTy = LLTy.getElementType();
 473
 474   // Pointer information was discarded. We'll need to coerce some register types
 475   // to avoid violating type constraints.
 476   LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();
 477
 478   assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());
 479
 480   if (DstEltTy == PartLLT) {
 481     // Vector was trivially scalarized.
 482
 483     if (RealDstEltTy.isPointer()) {
 484       for (Register Reg : Regs)
 485         MRI.setType(Reg, RealDstEltTy);
 486     }
 487
 488     B.buildBuildVector(OrigRegs[0], Regs);
 489   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
 490     // Deal with vector with 64-bit elements decomposed to 32-bit
 491     // registers. Need to create intermediate 64-bit elements.
 492     SmallVector<Register, 8> EltMerges;
 493     int PartsPerElt =
 494         divideCeil(DstEltTy.getSizeInBits(), PartLLT.getSizeInBits());
 495     LLT ExtendedPartTy = LLT::scalar(PartLLT.getSizeInBits() * PartsPerElt);
 496
 497     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {
 498       auto Merge =
 499           B.buildMergeLikeInstr(ExtendedPartTy, Regs.take_front(PartsPerElt));
 500       if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())
 501         Merge = B.buildTrunc(RealDstEltTy, Merge);
 502       // Fix the type in case this is really a vector of pointers.
 503       MRI.setType(Merge.getReg(0), RealDstEltTy);
 504       EltMerges.push_back(Merge.getReg(0));
 505       Regs = Regs.drop_front(PartsPerElt);
 506     }
 507
 508     B.buildBuildVector(OrigRegs[0], EltMerges);
 509   } else {
 510     // Vector was split, and elements promoted to a wider type.
 511     // FIXME: Should handle floating point promotions.
 512     unsigned NumElts = LLTy.getNumElements();
 513     LLT BVType = LLT::fixed_vector(NumElts, PartLLT);
 514
 515     Register BuildVec;
 516     if (NumElts == Regs.size())
 517       BuildVec = B.buildBuildVector(BVType, Regs).getReg(0);
 518     else {
 519       // Vector elements are packed in the inputs.
 520       // e.g. we have a <4 x s16> but 2 x s32 in regs.
 521       assert(NumElts > Regs.size());
 522       LLT SrcEltTy = MRI.getType(Regs[0]);
 523
 524       LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType();
 525
 526       // Input registers contain packed elements.
 527       // Determine how many elements per reg.
 528       assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);
 529       unsigned EltPerReg =
 530           (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());
 531
 532       SmallVector<Register, 0> BVRegs;
 533       BVRegs.reserve(Regs.size() * EltPerReg);
 534       for (Register R : Regs) {
 535         auto Unmerge = B.buildUnmerge(OriginalEltTy, R);
 536         for (unsigned K = 0; K < EltPerReg; ++K)
 537           BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0));
 538       }
 539
 540       // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces
 541       // for a <3 x s16> vector. We should have less than EltPerReg extra items.
 542       if (BVRegs.size() > NumElts) {
 543         assert((BVRegs.size() - NumElts) < EltPerReg);
 544         BVRegs.truncate(NumElts);
 545       }
 546       BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0);
 547     }
 548     B.buildTrunc(OrigRegs[0], BuildVec);
 549   }
 550 }
 551
 552 /// Create a sequence of instructions to expand the value in \p SrcReg (of type
 553 /// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should
 554 /// contain the type of scalar value extension if necessary.
 555 ///
 556 /// This is used for outgoing values (vregs to physregs)
 557 static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
 558                             Register SrcReg, LLT SrcTy, LLT PartTy,
 559                             unsigned ExtendOp = TargetOpcode::G_ANYEXT) {
 560   // We could just insert a regular copy, but this is unreachable at the moment.
 561   assert(SrcTy != PartTy && "identical part types shouldn't reach here");
 562
 563   const TypeSize PartSize = PartTy.getSizeInBits();
 564
 565   if (PartTy.isVector() == SrcTy.isVector() &&
 566       PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {
 567     assert(DstRegs.size() == 1);
 568     B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});
 569     return;
 570   }
 571
 572   if (SrcTy.isVector() && !PartTy.isVector() &&
 573       TypeSize::isKnownGT(PartSize, SrcTy.getElementType().getSizeInBits())) {
 574     // Vector was scalarized, and the elements extended.
 575     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);
 576     for (int i = 0, e = DstRegs.size(); i != e; ++i)
 577       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
 578     return;
 579   }
 580
 581   if (SrcTy.isVector() && PartTy.isVector() &&
 582       PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&
 583       ElementCount::isKnownLT(SrcTy.getElementCount(),
 584                               PartTy.getElementCount())) {
 585     // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32
 586     Register DstReg = DstRegs.front();
 587     B.buildPadVectorWithUndefElements(DstReg, SrcReg);
 588     return;
 589   }
 590
 591   LLT GCDTy = getGCDType(SrcTy, PartTy);
 592   if (GCDTy == PartTy) {
 593     // If this already evenly divisible, we can create a simple unmerge.
 594     B.buildUnmerge(DstRegs, SrcReg);
 595     return;
 596   }
 597
 598   if (SrcTy.isVector() && !PartTy.isVector() &&
 599       SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {
 600     LLT ExtTy =
 601         LLT::vector(SrcTy.getElementCount(),
 602                     LLT::scalar(PartTy.getScalarSizeInBits() * DstRegs.size() /
 603                                 SrcTy.getNumElements()));
 604     auto Ext = B.buildAnyExt(ExtTy, SrcReg);
 605     B.buildUnmerge(DstRegs, Ext);
 606     return;
 607   }
 608
 609   MachineRegisterInfo &MRI = *B.getMRI();
 610   LLT DstTy = MRI.getType(DstRegs[0]);
 611   LLT LCMTy = getCoverTy(SrcTy, PartTy);
 612
 613   if (PartTy.isVector() && LCMTy == PartTy) {
 614     assert(DstRegs.size() == 1);
 615     B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);
 616     return;
 617   }
 618
 619   const unsigned DstSize = DstTy.getSizeInBits();
 620   const unsigned SrcSize = SrcTy.getSizeInBits();
 621   unsigned CoveringSize = LCMTy.getSizeInBits();
 622
 623   Register UnmergeSrc = SrcReg;
 624
 625   if (!LCMTy.isVector() && CoveringSize != SrcSize) {
 626     // For scalars, it's common to be able to use a simple extension.
 627     if (SrcTy.isScalar() && DstTy.isScalar()) {
 628       CoveringSize = alignTo(SrcSize, DstSize);
 629       LLT CoverTy = LLT::scalar(CoveringSize);
 630       UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);
 631     } else {
 632       // Widen to the common type.
 633       // FIXME: This should respect the extend type
 634       Register Undef = B.buildUndef(SrcTy).getReg(0);
 635       SmallVector<Register, 8> MergeParts(1, SrcReg);
 636       for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)
 637         MergeParts.push_back(Undef);
 638       UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);
 639     }
 640   }
 641
 642   if (LCMTy.isVector() && CoveringSize != SrcSize)
 643     UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);
 644
 645   B.buildUnmerge(DstRegs, UnmergeSrc);
 646 }
 647
 648 bool CallLowering::determineAndHandleAssignments(
 649     ValueHandler &Handler, ValueAssigner &Assigner,
 650     SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,
 651     CallingConv::ID CallConv, bool IsVarArg,
 652     ArrayRef<Register> ThisReturnRegs) const {
 653   MachineFunction &MF = MIRBuilder.getMF();
 654   const Function &F = MF.getFunction();
 655   SmallVector<CCValAssign, 16> ArgLocs;
 656
 657   CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());
 658   if (!determineAssignments(Assigner, Args, CCInfo))
 659     return false;
 660
 661   return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,
 662                            ThisReturnRegs);
 663 }
 664
 665 static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {
 666   if (Flags.isSExt())
 667     return TargetOpcode::G_SEXT;
 668   if (Flags.isZExt())
 669     return TargetOpcode::G_ZEXT;
 670   return TargetOpcode::G_ANYEXT;
 671 }
 672
 673 bool CallLowering::determineAssignments(ValueAssigner &Assigner,
 674                                         SmallVectorImpl<ArgInfo> &Args,
 675                                         CCState &CCInfo) const {
 676   LLVMContext &Ctx = CCInfo.getContext();
 677   const CallingConv::ID CallConv = CCInfo.getCallingConv();
 678
 679   unsigned NumArgs = Args.size();
 680   for (unsigned i = 0; i != NumArgs; ++i) {
 681     EVT CurVT = EVT::getEVT(Args[i].Ty);
 682
 683     MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);
 684
 685     // If we need to split the type over multiple regs, check it's a scenario
 686     // we currently support.
 687     unsigned NumParts =
 688         TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);
 689
 690     if (NumParts == 1) {
 691       // Try to use the register type if we couldn't assign the VT.
 692       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
 693                              Args[i].Flags[0], CCInfo))
 694         return false;
 695       continue;
 696     }
 697
 698     // For incoming arguments (physregs to vregs), we could have values in
 699     // physregs (or memlocs) which we want to extract and copy to vregs.
 700     // During this, we might have to deal with the LLT being split across
 701     // multiple regs, so we have to record this information for later.
 702     //
 703     // If we have outgoing args, then we have the opposite case. We have a
 704     // vreg with an LLT which we want to assign to a physical location, and
 705     // we might have to record that the value has to be split later.
 706
 707     // We're handling an incoming arg which is split over multiple regs.
 708     // E.g. passing an s128 on AArch64.
 709     ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
 710     Args[i].Flags.clear();
 711
 712     for (unsigned Part = 0; Part < NumParts; ++Part) {
 713       ISD::ArgFlagsTy Flags = OrigFlags;
 714       if (Part == 0) {
 715         Flags.setSplit();
 716       } else {
 717         Flags.setOrigAlign(Align(1));
 718         if (Part == NumParts - 1)
 719           Flags.setSplitEnd();
 720       }
 721
 722       Args[i].Flags.push_back(Flags);
 723       if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],
 724                              Args[i].Flags[Part], CCInfo)) {
 725         // Still couldn't assign this smaller part type for some reason.
 726         return false;
 727       }
 728     }
 729   }
 730
 731   return true;
 732 }
 733
 734 bool CallLowering::handleAssignments(ValueHandler &Handler,
 735                                      SmallVectorImpl<ArgInfo> &Args,
 736                                      CCState &CCInfo,
 737                                      SmallVectorImpl<CCValAssign> &ArgLocs,
 738                                      MachineIRBuilder &MIRBuilder,
 739                                      ArrayRef<Register> ThisReturnRegs) const {
 740   MachineFunction &MF = MIRBuilder.getMF();
 741   MachineRegisterInfo &MRI = MF.getRegInfo();
 742   const Function &F = MF.getFunction();
 743   const DataLayout &DL = F.getDataLayout();
 744
 745   const unsigned NumArgs = Args.size();
 746
 747   // Stores thunks for outgoing register assignments. This is used so we delay
 748   // generating register copies until mem loc assignments are done. We do this
 749   // so that if the target is using the delayed stack protector feature, we can
 750   // find the split point of the block accurately. E.g. if we have:
 751   // G_STORE %val, %memloc
 752   // $x0 = COPY %foo
 753   // $x1 = COPY %bar
 754   // CALL func
 755   // ... then the split point for the block will correctly be at, and including,
 756   // the copy to $x0. If instead the G_STORE instruction immediately precedes
 757   // the CALL, then we'd prematurely choose the CALL as the split point, thus
 758   // generating a split block with a CALL that uses undefined physregs.
 759   SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;
 760
 761   for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {
 762     assert(j < ArgLocs.size() && "Skipped too many arg locs");
 763     CCValAssign &VA = ArgLocs[j];
 764     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
 765
 766     if (VA.needsCustom()) {
 767       std::function<void()> Thunk;
 768       unsigned NumArgRegs = Handler.assignCustomValue(
 769           Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);
 770       if (Thunk)
 771         DelayedOutgoingRegAssignments.emplace_back(Thunk);
 772       if (!NumArgRegs)
 773         return false;
 774       j += (NumArgRegs - 1);
 775       continue;
 776     }
 777
 778     auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();
 779
 780     const MVT ValVT = VA.getValVT();
 781     const MVT LocVT = VA.getLocVT();
 782
 783     const LLT LocTy(LocVT);
 784     const LLT ValTy(ValVT);
 785     const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;
 786     const EVT OrigVT = EVT::getEVT(Args[i].Ty);
 787     const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);
 788     const LLT PointerTy = LLT::pointer(
 789         AllocaAddressSpace, DL.getPointerSizeInBits(AllocaAddressSpace));
 790
 791     // Expected to be multiple regs for a single incoming arg.
 792     // There should be Regs.size() ArgLocs per argument.
 793     // This should be the same as getNumRegistersForCallingConv
 794     const unsigned NumParts = Args[i].Flags.size();
 795
 796     // Now split the registers into the assigned types.
 797     Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());
 798
 799     if (NumParts != 1 || NewLLT != OrigTy) {
 800       // If we can't directly assign the register, we need one or more
 801       // intermediate values.
 802       Args[i].Regs.resize(NumParts);
 803
 804       // When we have indirect parameter passing we are receiving a pointer,
 805       // that points to the actual value, so we need one "temporary" pointer.
 806       if (VA.getLocInfo() == CCValAssign::Indirect) {
 807         if (Handler.isIncomingArgumentHandler())
 808           Args[i].Regs[0] = MRI.createGenericVirtualRegister(PointerTy);
 809       } else {
 810         // For each split register, create and assign a vreg that will store
 811         // the incoming component of the larger value. These will later be
 812         // merged to form the final vreg.
 813         for (unsigned Part = 0; Part < NumParts; ++Part)
 814           Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);
 815       }
 816     }
 817
 818     assert((j + (NumParts - 1)) < ArgLocs.size() &&
 819            "Too many regs for number of args");
 820
 821     // Coerce into outgoing value types before register assignment.
 822     if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&
 823         VA.getLocInfo() != CCValAssign::Indirect) {
 824       assert(Args[i].OrigRegs.size() == 1);
 825       buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,
 826                       ValTy, extendOpFromFlags(Args[i].Flags[0]));
 827     }
 828
 829     bool IndirectParameterPassingHandled = false;
 830     bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);
 831     for (unsigned Part = 0; Part < NumParts; ++Part) {
 832       assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&
 833              "Only the first parameter should be processed when "
 834              "handling indirect passing!");
 835       Register ArgReg = Args[i].Regs[Part];
 836       // There should be Regs.size() ArgLocs per argument.
 837       unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;
 838       CCValAssign &VA = ArgLocs[j + Idx];
 839       const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];
 840
 841       // We found an indirect parameter passing, and we have an
 842       // OutgoingValueHandler as our handler (so we are at the call site or the
 843       // return value). In this case, start the construction of the following
 844       // GMIR, that is responsible for the preparation of indirect parameter
 845       // passing:
 846       //
 847       // %1(indirectly passed type) = The value to pass
 848       // %3(pointer) = G_FRAME_INDEX %stack.0
 849       // G_STORE %1, %3 :: (store (s128), align 8)
 850       //
 851       // After this GMIR, the remaining part of the loop body will decide how
 852       // to get the value to the caller and we break out of the loop.
 853       if (VA.getLocInfo() == CCValAssign::Indirect &&
 854           !Handler.isIncomingArgumentHandler()) {
 855         Align AlignmentForStored = DL.getPrefTypeAlign(Args[i].Ty);
 856         MachineFrameInfo &MFI = MF.getFrameInfo();
 857         // Get some space on the stack for the value, so later we can pass it
 858         // as a reference.
 859         int FrameIdx = MFI.CreateStackObject(OrigTy.getScalarSizeInBits(),
 860                                              AlignmentForStored, false);
 861         Register PointerToStackReg =
 862             MIRBuilder.buildFrameIndex(PointerTy, FrameIdx).getReg(0);
 863         MachinePointerInfo StackPointerMPO =
 864             MachinePointerInfo::getFixedStack(MF, FrameIdx);
 865         // Store the value in the previously created stack space.
 866         MIRBuilder.buildStore(Args[i].OrigRegs[Part], PointerToStackReg,
 867                               StackPointerMPO,
 868                               inferAlignFromPtrInfo(MF, StackPointerMPO));
 869
 870         ArgReg = PointerToStackReg;
 871         IndirectParameterPassingHandled = true;
 872       }
 873
 874       if (VA.isMemLoc() && !Flags.isByVal()) {
 875         // Individual pieces may have been spilled to the stack and others
 876         // passed in registers.
 877
 878         // TODO: The memory size may be larger than the value we need to
 879         // store. We may need to adjust the offset for big endian targets.
 880         LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);
 881
 882         MachinePointerInfo MPO;
 883         Register StackAddr =
 884             Handler.getStackAddress(VA.getLocInfo() == CCValAssign::Indirect
 885                                         ? PointerTy.getSizeInBytes()
 886                                         : MemTy.getSizeInBytes(),
 887                                     VA.getLocMemOffset(), MPO, Flags);
 888
 889         // Finish the handling of indirect passing from the passers
 890         // (OutgoingParameterHandler) side.
 891         // This branch is needed, so the pointer to the value is loaded onto the
 892         // stack.
 893         if (VA.getLocInfo() == CCValAssign::Indirect)
 894           Handler.assignValueToAddress(ArgReg, StackAddr, PointerTy, MPO, VA);
 895         else
 896           Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO,
 897                                        VA);
 898       } else if (VA.isMemLoc() && Flags.isByVal()) {
 899         assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");
 900
 901         if (Handler.isIncomingArgumentHandler()) {
 902           // We just need to copy the frame index value to the pointer.
 903           MachinePointerInfo MPO;
 904           Register StackAddr = Handler.getStackAddress(
 905               Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);
 906           MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);
 907         } else {
 908           // For outgoing byval arguments, insert the implicit copy byval
 909           // implies, such that writes in the callee do not modify the caller's
 910           // value.
 911           uint64_t MemSize = Flags.getByValSize();
 912           int64_t Offset = VA.getLocMemOffset();
 913
 914           MachinePointerInfo DstMPO;
 915           Register StackAddr =
 916               Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);
 917
 918           MachinePointerInfo SrcMPO(Args[i].OrigValue);
 919           if (!Args[i].OrigValue) {
 920             // We still need to accurately track the stack address space if we
 921             // don't know the underlying value.
 922             const LLT PtrTy = MRI.getType(StackAddr);
 923             SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());
 924           }
 925
 926           Align DstAlign = std::max(Flags.getNonZeroByValAlign(),
 927                                     inferAlignFromPtrInfo(MF, DstMPO));
 928
 929           Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),
 930                                     inferAlignFromPtrInfo(MF, SrcMPO));
 931
 932           Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],
 933                                      DstMPO, DstAlign, SrcMPO, SrcAlign,
 934                                      MemSize, VA);
 935         }
 936       } else if (i == 0 && !ThisReturnRegs.empty() &&
 937                  Handler.isIncomingArgumentHandler() &&
 938                  isTypeIsValidForThisReturn(ValVT)) {
 939         Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);
 940       } else if (Handler.isIncomingArgumentHandler()) {
 941         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
 942       } else {
 943         DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {
 944           Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
 945         });
 946       }
 947
 948       // Finish the handling of indirect parameter passing when receiving
 949       // the value (we are in the called function or the caller when receiving
 950       // the return value).
 951       if (VA.getLocInfo() == CCValAssign::Indirect &&
 952           Handler.isIncomingArgumentHandler()) {
 953         Align Alignment = DL.getABITypeAlign(Args[i].Ty);
 954         MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);
 955
 956         // Since we are doing indirect parameter passing, we know that the value
 957         // in the temporary register is not the value passed to the function,
 958         // but rather a pointer to that value. Let's load that value into the
 959         // virtual register where the parameter should go.
 960         MIRBuilder.buildLoad(Args[i].OrigRegs[0], Args[i].Regs[0], MPO,
 961                              Alignment);
 962
 963         IndirectParameterPassingHandled = true;
 964       }
 965
 966       if (IndirectParameterPassingHandled)
 967         break;
 968     }
 969
 970     // Now that all pieces have been assigned, re-pack the register typed values
 971     // into the original value typed registers. This is only necessary, when
 972     // the value was passed in multiple registers, not indirectly.
 973     if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&
 974         !IndirectParameterPassingHandled) {
 975       // Merge the split registers into the expected larger result vregs of
 976       // the original call.
 977       buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,
 978                         LocTy, Args[i].Flags[0]);
 979     }
 980
 981     j += NumParts - 1;
 982   }
 983   for (auto &Fn : DelayedOutgoingRegAssignments)
 984     Fn();
 985
 986   return true;
 987 }
 988
 989 void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,
 990                                    ArrayRef<Register> VRegs, Register DemoteReg,
 991                                    int FI) const {
 992   MachineFunction &MF = MIRBuilder.getMF();
 993   MachineRegisterInfo &MRI = MF.getRegInfo();
 994   const DataLayout &DL = MF.getDataLayout();
 995
 996   SmallVector<EVT, 4> SplitVTs;
 997   SmallVector<uint64_t, 4> Offsets;
 998   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
 999
1000   assert(VRegs.size() == SplitVTs.size());
1001
1002   unsigned NumValues = SplitVTs.size();
1003   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
1004   Type *RetPtrTy =
1005       PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());
1006   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);
1007
1008   MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
1009
1010   for (unsigned I = 0; I < NumValues; ++I) {
1011     Register Addr;
1012     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
1013     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1014                                         MRI.getType(VRegs[I]),
1015                                         commonAlignment(BaseAlign, Offsets[I]));
1016     MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);
1017   }
1018 }
1019
1020 void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
1021                                     ArrayRef<Register> VRegs,
1022                                     Register DemoteReg) const {
1023   MachineFunction &MF = MIRBuilder.getMF();
1024   MachineRegisterInfo &MRI = MF.getRegInfo();
1025   const DataLayout &DL = MF.getDataLayout();
1026
1027   SmallVector<EVT, 4> SplitVTs;
1028   SmallVector<uint64_t, 4> Offsets;
1029   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, &Offsets, 0);
1030
1031   assert(VRegs.size() == SplitVTs.size());
1032
1033   unsigned NumValues = SplitVTs.size();
1034   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
1035   unsigned AS = DL.getAllocaAddrSpace();
1036   LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getContext(), AS), DL);
1037
1038   MachinePointerInfo PtrInfo(AS);
1039
1040   for (unsigned I = 0; I < NumValues; ++I) {
1041     Register Addr;
1042     MIRBuilder.materializePtrAdd(Addr, DemoteReg, OffsetLLTy, Offsets[I]);
1043     auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
1044                                         MRI.getType(VRegs[I]),
1045                                         commonAlignment(BaseAlign, Offsets[I]));
1046     MIRBuilder.buildStore(VRegs[I], Addr, *MMO);
1047   }
1048 }
1049
1050 void CallLowering::insertSRetIncomingArgument(
1051     const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,
1052     MachineRegisterInfo &MRI, const DataLayout &DL) const {
1053   unsigned AS = DL.getAllocaAddrSpace();
1054   DemoteReg = MRI.createGenericVirtualRegister(
1055       LLT::pointer(AS, DL.getPointerSizeInBits(AS)));
1056
1057   Type *PtrTy = PointerType::get(F.getContext(), AS);
1058
1059   SmallVector<EVT, 1> ValueVTs;
1060   ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);
1061
1062   // NOTE: Assume that a pointer won't get split into more than one VT.
1063   assert(ValueVTs.size() == 1);
1064
1065   ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),
1066                     ArgInfo::NoArgIndex);
1067   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);
1068   DemoteArg.Flags[0].setSRet();
1069   SplitArgs.insert(SplitArgs.begin(), DemoteArg);
1070 }
1071
1072 void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,
1073                                               const CallBase &CB,
1074                                               CallLoweringInfo &Info) const {
1075   const DataLayout &DL = MIRBuilder.getDataLayout();
1076   Type *RetTy = CB.getType();
1077   unsigned AS = DL.getAllocaAddrSpace();
1078   LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
1079
1080   int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(
1081       DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);
1082
1083   Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);
1084   ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy->getContext(), AS),
1085                     ArgInfo::NoArgIndex);
1086   setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);
1087   DemoteArg.Flags[0].setSRet();
1088
1089   Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);
1090   Info.DemoteStackIndex = FI;
1091   Info.DemoteRegister = DemoteReg;
1092 }
1093
1094 bool CallLowering::checkReturn(CCState &CCInfo,
1095                                SmallVectorImpl<BaseArgInfo> &Outs,
1096                                CCAssignFn *Fn) const {
1097   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
1098     MVT VT = MVT::getVT(Outs[I].Ty);
1099     if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
1100       return false;
1101   }
1102   return true;
1103 }
1104
1105 void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,
1106                                  AttributeList Attrs,
1107                                  SmallVectorImpl<BaseArgInfo> &Outs,
1108                                  const DataLayout &DL) const {
1109   LLVMContext &Context = RetTy->getContext();
1110   ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1111
1112   SmallVector<EVT, 4> SplitVTs;
1113   ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);
1114   addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);
1115
1116   for (EVT VT : SplitVTs) {
1117     unsigned NumParts =
1118         TLI->getNumRegistersForCallingConv(Context, CallConv, VT);
1119     MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);
1120     Type *PartTy = EVT(RegVT).getTypeForEVT(Context);
1121
1122     for (unsigned I = 0; I < NumParts; ++I) {
1123       Outs.emplace_back(PartTy, Flags);
1124     }
1125   }
1126 }
1127
1128 bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {
1129   const auto &F = MF.getFunction();
1130   Type *ReturnType = F.getReturnType();
1131   CallingConv::ID CallConv = F.getCallingConv();
1132
1133   SmallVector<BaseArgInfo, 4> SplitArgs;
1134   getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,
1135                 MF.getDataLayout());
1136   return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());
1137 }
1138
1139 bool CallLowering::parametersInCSRMatch(
1140     const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,
1141     const SmallVectorImpl<CCValAssign> &OutLocs,
1142     const SmallVectorImpl<ArgInfo> &OutArgs) const {
1143   for (unsigned i = 0; i < OutLocs.size(); ++i) {
1144     const auto &ArgLoc = OutLocs[i];
1145     // If it's not a register, it's fine.
1146     if (!ArgLoc.isRegLoc())
1147       continue;
1148
1149     MCRegister PhysReg = ArgLoc.getLocReg();
1150
1151     // Only look at callee-saved registers.
1152     if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))
1153       continue;
1154
1155     LLVM_DEBUG(
1156         dbgs()
1157         << "... Call has an argument passed in a callee-saved register.\n");
1158
1159     // Check if it was copied from.
1160     const ArgInfo &OutInfo = OutArgs[i];
1161
1162     if (OutInfo.Regs.size() > 1) {
1163       LLVM_DEBUG(
1164           dbgs() << "... Cannot handle arguments in multiple registers.\n");
1165       return false;
1166     }
1167
1168     // Check if we copy the register, walking through copies from virtual
1169     // registers. Note that getDefIgnoringCopies does not ignore copies from
1170     // physical registers.
1171     MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);
1172     if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {
1173       LLVM_DEBUG(
1174           dbgs()
1175           << "... Parameter was not copied into a VReg, cannot tail call.\n");
1176       return false;
1177     }
1178
1179     // Got a copy. Verify that it's the same as the register we want.
1180     Register CopyRHS = RegDef->getOperand(1).getReg();
1181     if (CopyRHS != PhysReg) {
1182       LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
1183                            "VReg, cannot tail call.\n");
1184       return false;
1185     }
1186   }
1187
1188   return true;
1189 }
1190
1191 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
1192                                      MachineFunction &MF,
1193                                      SmallVectorImpl<ArgInfo> &InArgs,
1194                                      ValueAssigner &CalleeAssigner,
1195                                      ValueAssigner &CallerAssigner) const {
1196   const Function &F = MF.getFunction();
1197   CallingConv::ID CalleeCC = Info.CallConv;
1198   CallingConv::ID CallerCC = F.getCallingConv();
1199
1200   if (CallerCC == CalleeCC)
1201     return true;
1202
1203   SmallVector<CCValAssign, 16> ArgLocs1;
1204   CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());
1205   if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))
1206     return false;
1207
1208   SmallVector<CCValAssign, 16> ArgLocs2;
1209   CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());
1210   if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))
1211     return false;
1212
1213   // We need the argument locations to match up exactly. If there's more in
1214   // one than the other, then we are done.
1215   if (ArgLocs1.size() != ArgLocs2.size())
1216     return false;
1217
1218   // Make sure that each location is passed in exactly the same way.
1219   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
1220     const CCValAssign &Loc1 = ArgLocs1[i];
1221     const CCValAssign &Loc2 = ArgLocs2[i];
1222
1223     // We need both of them to be the same. So if one is a register and one
1224     // isn't, we're done.
1225     if (Loc1.isRegLoc() != Loc2.isRegLoc())
1226       return false;
1227
1228     if (Loc1.isRegLoc()) {
1229       // If they don't have the same register location, we're done.
1230       if (Loc1.getLocReg() != Loc2.getLocReg())
1231         return false;
1232
1233       // They matched, so we can move to the next ArgLoc.
1234       continue;
1235     }
1236
1237     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
1238     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
1239       return false;
1240   }
1241
1242   return true;
1243 }
1244
1245 LLT CallLowering::ValueHandler::getStackValueStoreType(
1246     const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {
1247   const MVT ValVT = VA.getValVT();
1248   if (ValVT != MVT::iPTR) {
1249     LLT ValTy(ValVT);
1250
1251     // We lost the pointeriness going through CCValAssign, so try to restore it
1252     // based on the flags.
1253     if (Flags.isPointer()) {
1254       LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),
1255                                ValTy.getScalarSizeInBits());
1256       if (ValVT.isVector())
1257         return LLT::vector(ValTy.getElementCount(), PtrTy);
1258       return PtrTy;
1259     }
1260
1261     return ValTy;
1262   }
1263
1264   unsigned AddrSpace = Flags.getPointerAddrSpace();
1265   return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));
1266 }
1267
1268 void CallLowering::ValueHandler::copyArgumentMemory(
1269     const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
1270     const MachinePointerInfo &DstPtrInfo, Align DstAlign,
1271     const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,
1272     CCValAssign &VA) const {
1273   MachineFunction &MF = MIRBuilder.getMF();
1274   MachineMemOperand *SrcMMO = MF.getMachineMemOperand(
1275       SrcPtrInfo,
1276       MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,
1277       SrcAlign);
1278
1279   MachineMemOperand *DstMMO = MF.getMachineMemOperand(
1280       DstPtrInfo,
1281       MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,
1282       MemSize, DstAlign);
1283
1284   const LLT PtrTy = MRI.getType(DstPtr);
1285   const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());
1286
1287   auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);
1288   MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);
1289 }
1290
1291 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
1292                                                     const CCValAssign &VA,
1293                                                     unsigned MaxSizeBits) {
1294   LLT LocTy{VA.getLocVT()};
1295   LLT ValTy{VA.getValVT()};
1296
1297   if (LocTy.getSizeInBits() == ValTy.getSizeInBits())
1298     return ValReg;
1299
1300   if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {
1301     if (MaxSizeBits <= ValTy.getSizeInBits())
1302       return ValReg;
1303     LocTy = LLT::scalar(MaxSizeBits);
1304   }
1305
1306   const LLT ValRegTy = MRI.getType(ValReg);
1307   if (ValRegTy.isPointer()) {
1308     // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so
1309     // we have to cast to do the extension.
1310     LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());
1311     ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);
1312   }
1313
1314   switch (VA.getLocInfo()) {
1315   default:
1316     break;
1317   case CCValAssign::Full:
1318   case CCValAssign::BCvt:
1319     // FIXME: bitconverting between vector types may or may not be a
1320     // nop in big-endian situations.
1321     return ValReg;
1322   case CCValAssign::AExt: {
1323     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
1324     return MIB.getReg(0);
1325   }
1326   case CCValAssign::SExt: {
1327     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1328     MIRBuilder.buildSExt(NewReg, ValReg);
1329     return NewReg;
1330   }
1331   case CCValAssign::ZExt: {
1332     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
1333     MIRBuilder.buildZExt(NewReg, ValReg);
1334     return NewReg;
1335   }
1336   }
1337   llvm_unreachable("unable to extend register");
1338 }
1339
1340 void CallLowering::ValueAssigner::anchor() {}
1341
1342 Register CallLowering::IncomingValueHandler::buildExtensionHint(
1343     const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {
1344   switch (VA.getLocInfo()) {
1345   case CCValAssign::LocInfo::ZExt: {
1346     return MIRBuilder
1347         .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1348                          NarrowTy.getScalarSizeInBits())
1349         .getReg(0);
1350   }
1351   case CCValAssign::LocInfo::SExt: {
1352     return MIRBuilder
1353         .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,
1354                          NarrowTy.getScalarSizeInBits())
1355         .getReg(0);
1356     break;
1357   }
1358   default:
1359     return SrcReg;
1360   }
1361 }
1362
1363 /// Check if we can use a basic COPY instruction between the two types.
1364 ///
1365 /// We're currently building on top of the infrastructure using MVT, which loses
1366 /// pointer information in the CCValAssign. We accept copies from physical
1367 /// registers that have been reported as integers if it's to an equivalent sized
1368 /// pointer LLT.
1369 static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {
1370   if (SrcTy == DstTy)
1371     return true;
1372
1373   if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
1374     return false;
1375
1376   SrcTy = SrcTy.getScalarType();
1377   DstTy = DstTy.getScalarType();
1378
1379   return (SrcTy.isPointer() && DstTy.isScalar()) ||
1380          (DstTy.isPointer() && SrcTy.isScalar());
1381 }
1382
1383 void CallLowering::IncomingValueHandler::assignValueToReg(
1384     Register ValVReg, Register PhysReg, const CCValAssign &VA) {
1385   const MVT LocVT = VA.getLocVT();
1386   const LLT LocTy(LocVT);
1387   const LLT RegTy = MRI.getType(ValVReg);
1388
1389   if (isCopyCompatibleType(RegTy, LocTy)) {
1390     MIRBuilder.buildCopy(ValVReg, PhysReg);
1391     return;
1392   }
1393
1394   auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);
1395   auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);
1396   MIRBuilder.buildTrunc(ValVReg, Hint);
1397 }