lib/CodeGen/GlobalISel/CallLowering.cpp

   1 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file implements some simple delegations needed for call lowering.
  11 ///
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/CodeGen/Analysis.h"
  15 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
  16 #include "llvm/CodeGen/GlobalISel/Utils.h"
  17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  18 #include "llvm/CodeGen/MachineOperand.h"
  19 #include "llvm/CodeGen/MachineRegisterInfo.h"
  20 #include "llvm/CodeGen/TargetLowering.h"
  21 #include "llvm/IR/DataLayout.h"
  22 #include "llvm/IR/Instructions.h"
  23 #include "llvm/IR/LLVMContext.h"
  24 #include "llvm/IR/Module.h"
  25
  26 #define DEBUG_TYPE "call-lowering"
  27
  28 using namespace llvm;
  29
  30 void CallLowering::anchor() {}
  31
  32 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
  33                              ArrayRef<Register> ResRegs,
  34                              ArrayRef<ArrayRef<Register>> ArgRegs,
  35                              Register SwiftErrorVReg,
  36                              std::function<unsigned()> GetCalleeReg) const {
  37   CallLoweringInfo Info;
  38   auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
  39
  40   // First step is to marshall all the function's parameters into the correct
  41   // physregs and memory locations. Gather the sequence of argument types that
  42   // we'll pass to the assigner function.
  43   unsigned i = 0;
  44   unsigned NumFixedArgs = CS.getFunctionType()->getNumParams();
  45   for (auto &Arg : CS.args()) {
  46     ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{},
  47                     i < NumFixedArgs};
  48     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS);
  49     Info.OrigArgs.push_back(OrigArg);
  50     ++i;
  51   }
  52
  53   if (const Function *F = CS.getCalledFunction())
  54     Info.Callee = MachineOperand::CreateGA(F, 0);
  55   else
  56     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
  57
  58   Info.OrigRet = ArgInfo{ResRegs, CS.getType(), ISD::ArgFlagsTy{}};
  59   if (!Info.OrigRet.Ty->isVoidTy())
  60     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS);
  61
  62   Info.KnownCallees =
  63       CS.getInstruction()->getMetadata(LLVMContext::MD_callees);
  64   Info.CallConv = CS.getCallingConv();
  65   Info.SwiftErrorVReg = SwiftErrorVReg;
  66   Info.IsMustTailCall = CS.isMustTailCall();
  67   Info.IsTailCall = CS.isTailCall() &&
  68                     isInTailCallPosition(CS, MIRBuilder.getMF().getTarget());
  69   Info.IsVarArg = CS.getFunctionType()->isVarArg();
  70   return lowerCall(MIRBuilder, Info);
  71 }
  72
  73 template <typename FuncInfoTy>
  74 void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
  75                                const DataLayout &DL,
  76                                const FuncInfoTy &FuncInfo) const {
  77   auto &Flags = Arg.Flags[0];
  78   const AttributeList &Attrs = FuncInfo.getAttributes();
  79   if (Attrs.hasAttribute(OpIdx, Attribute::ZExt))
  80     Flags.setZExt();
  81   if (Attrs.hasAttribute(OpIdx, Attribute::SExt))
  82     Flags.setSExt();
  83   if (Attrs.hasAttribute(OpIdx, Attribute::InReg))
  84     Flags.setInReg();
  85   if (Attrs.hasAttribute(OpIdx, Attribute::StructRet))
  86     Flags.setSRet();
  87   if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf))
  88     Flags.setSwiftSelf();
  89   if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError))
  90     Flags.setSwiftError();
  91   if (Attrs.hasAttribute(OpIdx, Attribute::ByVal))
  92     Flags.setByVal();
  93   if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca))
  94     Flags.setInAlloca();
  95
  96   if (Flags.isByVal() || Flags.isInAlloca()) {
  97     Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
  98
  99     auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType();
 100     Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy));
 101
 102     // For ByVal, alignment should be passed from FE.  BE will guess if
 103     // this info is not there but there are cases it cannot get right.
 104     unsigned FrameAlign;
 105     if (FuncInfo.getParamAlignment(OpIdx - 2))
 106       FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2);
 107     else
 108       FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
 109     Flags.setByValAlign(Align(FrameAlign));
 110   }
 111   if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
 112     Flags.setNest();
 113   Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
 114 }
 115
 116 template void
 117 CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 118                                     const DataLayout &DL,
 119                                     const Function &FuncInfo) const;
 120
 121 template void
 122 CallLowering::setArgFlags<CallInst>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
 123                                     const DataLayout &DL,
 124                                     const CallInst &FuncInfo) const;
 125
 126 Register CallLowering::packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy,
 127                                 MachineIRBuilder &MIRBuilder) const {
 128   assert(SrcRegs.size() > 1 && "Nothing to pack");
 129
 130   const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
 131   MachineRegisterInfo *MRI = MIRBuilder.getMRI();
 132
 133   LLT PackedLLT = getLLTForType(*PackedTy, DL);
 134
 135   SmallVector<LLT, 8> LLTs;
 136   SmallVector<uint64_t, 8> Offsets;
 137   computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
 138   assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch");
 139
 140   Register Dst = MRI->createGenericVirtualRegister(PackedLLT);
 141   MIRBuilder.buildUndef(Dst);
 142   for (unsigned i = 0; i < SrcRegs.size(); ++i) {
 143     Register NewDst = MRI->createGenericVirtualRegister(PackedLLT);
 144     MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]);
 145     Dst = NewDst;
 146   }
 147
 148   return Dst;
 149 }
 150
 151 void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
 152                               Type *PackedTy,
 153                               MachineIRBuilder &MIRBuilder) const {
 154   assert(DstRegs.size() > 1 && "Nothing to unpack");
 155
 156   const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
 157
 158   SmallVector<LLT, 8> LLTs;
 159   SmallVector<uint64_t, 8> Offsets;
 160   computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
 161   assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch");
 162
 163   for (unsigned i = 0; i < DstRegs.size(); ++i)
 164     MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]);
 165 }
 166
 167 bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder,
 168                                      SmallVectorImpl<ArgInfo> &Args,
 169                                      ValueHandler &Handler) const {
 170   MachineFunction &MF = MIRBuilder.getMF();
 171   const Function &F = MF.getFunction();
 172   SmallVector<CCValAssign, 16> ArgLocs;
 173   CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
 174   return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler);
 175 }
 176
 177 bool CallLowering::handleAssignments(CCState &CCInfo,
 178                                      SmallVectorImpl<CCValAssign> &ArgLocs,
 179                                      MachineIRBuilder &MIRBuilder,
 180                                      SmallVectorImpl<ArgInfo> &Args,
 181                                      ValueHandler &Handler) const {
 182   MachineFunction &MF = MIRBuilder.getMF();
 183   const Function &F = MF.getFunction();
 184   const DataLayout &DL = F.getParent()->getDataLayout();
 185
 186   unsigned NumArgs = Args.size();
 187   for (unsigned i = 0; i != NumArgs; ++i) {
 188     MVT CurVT = MVT::getVT(Args[i].Ty);
 189     if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i],
 190                           Args[i].Flags[0], CCInfo)) {
 191       if (!CurVT.isValid())
 192         return false;
 193       MVT NewVT = TLI->getRegisterTypeForCallingConv(
 194           F.getContext(), F.getCallingConv(), EVT(CurVT));
 195
 196       // If we need to split the type over multiple regs, check it's a scenario
 197       // we currently support.
 198       unsigned NumParts = TLI->getNumRegistersForCallingConv(
 199           F.getContext(), F.getCallingConv(), CurVT);
 200       if (NumParts > 1) {
 201         // For now only handle exact splits.
 202         if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits())
 203           return false;
 204       }
 205
 206       // For incoming arguments (physregs to vregs), we could have values in
 207       // physregs (or memlocs) which we want to extract and copy to vregs.
 208       // During this, we might have to deal with the LLT being split across
 209       // multiple regs, so we have to record this information for later.
 210       //
 211       // If we have outgoing args, then we have the opposite case. We have a
 212       // vreg with an LLT which we want to assign to a physical location, and
 213       // we might have to record that the value has to be split later.
 214       if (Handler.isIncomingArgumentHandler()) {
 215         if (NumParts == 1) {
 216           // Try to use the register type if we couldn't assign the VT.
 217           if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
 218                                 Args[i].Flags[0], CCInfo))
 219             return false;
 220         } else {
 221           // We're handling an incoming arg which is split over multiple regs.
 222           // E.g. passing an s128 on AArch64.
 223           ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
 224           Args[i].OrigRegs.push_back(Args[i].Regs[0]);
 225           Args[i].Regs.clear();
 226           Args[i].Flags.clear();
 227           LLT NewLLT = getLLTForMVT(NewVT);
 228           // For each split register, create and assign a vreg that will store
 229           // the incoming component of the larger value. These will later be
 230           // merged to form the final vreg.
 231           for (unsigned Part = 0; Part < NumParts; ++Part) {
 232             Register Reg =
 233                 MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT);
 234             ISD::ArgFlagsTy Flags = OrigFlags;
 235             if (Part == 0) {
 236               Flags.setSplit();
 237             } else {
 238               Flags.setOrigAlign(Align::None());
 239               if (Part == NumParts - 1)
 240                 Flags.setSplitEnd();
 241             }
 242             Args[i].Regs.push_back(Reg);
 243             Args[i].Flags.push_back(Flags);
 244             if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full,
 245                                   Args[i], Args[i].Flags[Part], CCInfo)) {
 246               // Still couldn't assign this smaller part type for some reason.
 247               return false;
 248             }
 249           }
 250         }
 251       } else {
 252         // Handling an outgoing arg that might need to be split.
 253         if (NumParts < 2)
 254           return false; // Don't know how to deal with this type combination.
 255
 256         // This type is passed via multiple registers in the calling convention.
 257         // We need to extract the individual parts.
 258         Register LargeReg = Args[i].Regs[0];
 259         LLT SmallTy = LLT::scalar(NewVT.getSizeInBits());
 260         auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg);
 261         assert(Unmerge->getNumOperands() == NumParts + 1);
 262         ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
 263         // We're going to replace the regs and flags with the split ones.
 264         Args[i].Regs.clear();
 265         Args[i].Flags.clear();
 266         for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) {
 267           ISD::ArgFlagsTy Flags = OrigFlags;
 268           if (PartIdx == 0) {
 269             Flags.setSplit();
 270           } else {
 271             Flags.setOrigAlign(Align::None());
 272             if (PartIdx == NumParts - 1)
 273               Flags.setSplitEnd();
 274           }
 275           Args[i].Regs.push_back(Unmerge.getReg(PartIdx));
 276           Args[i].Flags.push_back(Flags);
 277           if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full,
 278                                 Args[i], Args[i].Flags[PartIdx], CCInfo))
 279             return false;
 280         }
 281       }
 282     }
 283   }
 284
 285   for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) {
 286     assert(j < ArgLocs.size() && "Skipped too many arg locs");
 287
 288     CCValAssign &VA = ArgLocs[j];
 289     assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
 290
 291     if (VA.needsCustom()) {
 292       j += Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j));
 293       continue;
 294     }
 295
 296     // FIXME: Pack registers if we have more than one.
 297     Register ArgReg = Args[i].Regs[0];
 298
 299     MVT OrigVT = MVT::getVT(Args[i].Ty);
 300     MVT VAVT = VA.getValVT();
 301     if (VA.isRegLoc()) {
 302       if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
 303         if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
 304           // Expected to be multiple regs for a single incoming arg.
 305           unsigned NumArgRegs = Args[i].Regs.size();
 306           if (NumArgRegs < 2)
 307             return false;
 308
 309           assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
 310                  "Too many regs for number of args");
 311           for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
 312             // There should be Regs.size() ArgLocs per argument.
 313             VA = ArgLocs[j + Part];
 314             Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
 315           }
 316           j += NumArgRegs - 1;
 317           // Merge the split registers into the expected larger result vreg
 318           // of the original call.
 319           MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
 320           continue;
 321         }
 322         const LLT VATy(VAVT);
 323         Register NewReg =
 324             MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
 325         Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
 326         // If it's a vector type, we either need to truncate the elements
 327         // or do an unmerge to get the lower block of elements.
 328         if (VATy.isVector() &&
 329             VATy.getNumElements() > OrigVT.getVectorNumElements()) {
 330           const LLT OrigTy(OrigVT);
 331           // Just handle the case where the VA type is 2 * original type.
 332           if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
 333             LLVM_DEBUG(dbgs()
 334                        << "Incoming promoted vector arg has too many elts");
 335             return false;
 336           }
 337           auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
 338           MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
 339         } else {
 340           MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
 341         }
 342       } else if (!Handler.isIncomingArgumentHandler()) {
 343         assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() &&
 344                "Too many regs for number of args");
 345         // This is an outgoing argument that might have been split.
 346         for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) {
 347           // There should be Regs.size() ArgLocs per argument.
 348           VA = ArgLocs[j + Part];
 349           Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
 350         }
 351         j += Args[i].Regs.size() - 1;
 352       } else {
 353         Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
 354       }
 355     } else if (VA.isMemLoc()) {
 356       // Don't currently support loading/storing a type that needs to be split
 357       // to the stack. Should be easy, just not implemented yet.
 358       if (Args[i].Regs.size() > 1) {
 359         LLVM_DEBUG(
 360             dbgs()
 361             << "Load/store a split arg to/from the stack not implemented yet");
 362         return false;
 363       }
 364       MVT VT = MVT::getVT(Args[i].Ty);
 365       unsigned Size = VT == MVT::iPTR ? DL.getPointerSize()
 366                                       : alignTo(VT.getSizeInBits(), 8) / 8;
 367       unsigned Offset = VA.getLocMemOffset();
 368       MachinePointerInfo MPO;
 369       Register StackAddr = Handler.getStackAddress(Size, Offset, MPO);
 370       Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA);
 371     } else {
 372       // FIXME: Support byvals and other weirdness
 373       return false;
 374     }
 375   }
 376   return true;
 377 }
 378
 379 bool CallLowering::analyzeArgInfo(CCState &CCState,
 380                                   SmallVectorImpl<ArgInfo> &Args,
 381                                   CCAssignFn &AssignFnFixed,
 382                                   CCAssignFn &AssignFnVarArg) const {
 383   for (unsigned i = 0, e = Args.size(); i < e; ++i) {
 384     MVT VT = MVT::getVT(Args[i].Ty);
 385     CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
 386     if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
 387       // Bail out on anything we can't handle.
 388       LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
 389                         << " (arg number = " << i << "\n");
 390       return false;
 391     }
 392   }
 393   return true;
 394 }
 395
 396 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
 397                                      MachineFunction &MF,
 398                                      SmallVectorImpl<ArgInfo> &InArgs,
 399                                      CCAssignFn &CalleeAssignFnFixed,
 400                                      CCAssignFn &CalleeAssignFnVarArg,
 401                                      CCAssignFn &CallerAssignFnFixed,
 402                                      CCAssignFn &CallerAssignFnVarArg) const {
 403   const Function &F = MF.getFunction();
 404   CallingConv::ID CalleeCC = Info.CallConv;
 405   CallingConv::ID CallerCC = F.getCallingConv();
 406
 407   if (CallerCC == CalleeCC)
 408     return true;
 409
 410   SmallVector<CCValAssign, 16> ArgLocs1;
 411   CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
 412   if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
 413                       CalleeAssignFnVarArg))
 414     return false;
 415
 416   SmallVector<CCValAssign, 16> ArgLocs2;
 417   CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
 418   if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
 419                       CalleeAssignFnVarArg))
 420     return false;
 421
 422   // We need the argument locations to match up exactly. If there's more in
 423   // one than the other, then we are done.
 424   if (ArgLocs1.size() != ArgLocs2.size())
 425     return false;
 426
 427   // Make sure that each location is passed in exactly the same way.
 428   for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
 429     const CCValAssign &Loc1 = ArgLocs1[i];
 430     const CCValAssign &Loc2 = ArgLocs2[i];
 431
 432     // We need both of them to be the same. So if one is a register and one
 433     // isn't, we're done.
 434     if (Loc1.isRegLoc() != Loc2.isRegLoc())
 435       return false;
 436
 437     if (Loc1.isRegLoc()) {
 438       // If they don't have the same register location, we're done.
 439       if (Loc1.getLocReg() != Loc2.getLocReg())
 440         return false;
 441
 442       // They matched, so we can move to the next ArgLoc.
 443       continue;
 444     }
 445
 446     // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
 447     if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
 448       return false;
 449   }
 450
 451   return true;
 452 }
 453
 454 Register CallLowering::ValueHandler::extendRegister(Register ValReg,
 455                                                     CCValAssign &VA) {
 456   LLT LocTy{VA.getLocVT()};
 457   if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits())
 458     return ValReg;
 459   switch (VA.getLocInfo()) {
 460   default: break;
 461   case CCValAssign::Full:
 462   case CCValAssign::BCvt:
 463     // FIXME: bitconverting between vector types may or may not be a
 464     // nop in big-endian situations.
 465     return ValReg;
 466   case CCValAssign::AExt: {
 467     auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
 468     return MIB->getOperand(0).getReg();
 469   }
 470   case CCValAssign::SExt: {
 471     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
 472     MIRBuilder.buildSExt(NewReg, ValReg);
 473     return NewReg;
 474   }
 475   case CCValAssign::ZExt: {
 476     Register NewReg = MRI.createGenericVirtualRegister(LocTy);
 477     MIRBuilder.buildZExt(NewReg, ValReg);
 478     return NewReg;
 479   }
 480   }
 481   llvm_unreachable("unable to extend register");
 482 }
 483
 484 void CallLowering::ValueHandler::anchor() {}