llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp

   1 //===-- SystemZFrameLowering.cpp - Frame lowering for SystemZ -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "SystemZFrameLowering.h"
  10 #include "SystemZCallingConv.h"
  11 #include "SystemZInstrInfo.h"
  12 #include "SystemZMachineFunctionInfo.h"
  13 #include "SystemZRegisterInfo.h"
  14 #include "SystemZSubtarget.h"
  15 #include "llvm/CodeGen/LivePhysRegs.h"
  16 #include "llvm/CodeGen/MachineModuleInfo.h"
  17 #include "llvm/CodeGen/MachineRegisterInfo.h"
  18 #include "llvm/CodeGen/RegisterScavenging.h"
  19 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/Target/TargetMachine.h"
  22
  23 using namespace llvm;
  24
  25 namespace {
  26 // The ABI-defined register save slots, relative to the CFA (i.e.
  27 // incoming stack pointer + SystemZMC::ELFCallFrameSize).
  28 static const TargetFrameLowering::SpillSlot ELFSpillOffsetTable[] = {
  29   { SystemZ::R2D,  0x10 },
  30   { SystemZ::R3D,  0x18 },
  31   { SystemZ::R4D,  0x20 },
  32   { SystemZ::R5D,  0x28 },
  33   { SystemZ::R6D,  0x30 },
  34   { SystemZ::R7D,  0x38 },
  35   { SystemZ::R8D,  0x40 },
  36   { SystemZ::R9D,  0x48 },
  37   { SystemZ::R10D, 0x50 },
  38   { SystemZ::R11D, 0x58 },
  39   { SystemZ::R12D, 0x60 },
  40   { SystemZ::R13D, 0x68 },
  41   { SystemZ::R14D, 0x70 },
  42   { SystemZ::R15D, 0x78 },
  43   { SystemZ::F0D,  0x80 },
  44   { SystemZ::F2D,  0x88 },
  45   { SystemZ::F4D,  0x90 },
  46   { SystemZ::F6D,  0x98 }
  47 };
  48
  49 static const TargetFrameLowering::SpillSlot XPLINKSpillOffsetTable[] = {
  50     {SystemZ::R4D, 0x00},  {SystemZ::R5D, 0x08},  {SystemZ::R6D, 0x10},
  51     {SystemZ::R7D, 0x18},  {SystemZ::R8D, 0x20},  {SystemZ::R9D, 0x28},
  52     {SystemZ::R10D, 0x30}, {SystemZ::R11D, 0x38}, {SystemZ::R12D, 0x40},
  53     {SystemZ::R13D, 0x48}, {SystemZ::R14D, 0x50}, {SystemZ::R15D, 0x58}};
  54 } // end anonymous namespace
  55
  56 SystemZFrameLowering::SystemZFrameLowering(StackDirection D, Align StackAl,
  57                                            int LAO, Align TransAl,
  58                                            bool StackReal, unsigned PointerSize)
  59     : TargetFrameLowering(D, StackAl, LAO, TransAl, StackReal),
  60       PointerSize(PointerSize) {}
  61
  62 std::unique_ptr<SystemZFrameLowering>
  63 SystemZFrameLowering::create(const SystemZSubtarget &STI) {
  64   unsigned PtrSz =
  65       STI.getTargetLowering()->getTargetMachine().getPointerSize(0);
  66   if (STI.isTargetXPLINK64())
  67     return std::make_unique<SystemZXPLINKFrameLowering>(PtrSz);
  68   return std::make_unique<SystemZELFFrameLowering>(PtrSz);
  69 }
  70
  71 namespace {
  72 struct SZFrameSortingObj {
  73   bool IsValid = false;     // True if we care about this Object.
  74   uint32_t ObjectIndex = 0; // Index of Object into MFI list.
  75   uint64_t ObjectSize = 0;  // Size of Object in bytes.
  76   uint32_t D12Count = 0;    // 12-bit displacement only.
  77   uint32_t DPairCount = 0;  // 12 or 20 bit displacement.
  78 };
  79 typedef std::vector<SZFrameSortingObj> SZFrameObjVec;
  80 } // namespace
  81
  82 // TODO: Move to base class.
  83 void SystemZELFFrameLowering::orderFrameObjects(
  84     const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
  85   const MachineFrameInfo &MFI = MF.getFrameInfo();
  86   auto *TII = MF.getSubtarget<SystemZSubtarget>().getInstrInfo();
  87
  88   // Make a vector of sorting objects to track all MFI objects and mark those
  89   // to be sorted as valid.
  90   if (ObjectsToAllocate.size() <= 1)
  91     return;
  92   SZFrameObjVec SortingObjects(MFI.getObjectIndexEnd());
  93   for (auto &Obj : ObjectsToAllocate) {
  94     SortingObjects[Obj].IsValid = true;
  95     SortingObjects[Obj].ObjectIndex = Obj;
  96     SortingObjects[Obj].ObjectSize = MFI.getObjectSize(Obj);
  97   }
  98
  99   // Examine uses for each object and record short (12-bit) and "pair"
 100   // displacement types.
 101   for (auto &MBB : MF)
 102     for (auto &MI : MBB) {
 103       if (MI.isDebugInstr())
 104         continue;
 105       for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
 106         const MachineOperand &MO = MI.getOperand(I);
 107         if (!MO.isFI())
 108           continue;
 109         int Index = MO.getIndex();
 110         if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
 111             SortingObjects[Index].IsValid) {
 112           if (TII->hasDisplacementPairInsn(MI.getOpcode()))
 113             SortingObjects[Index].DPairCount++;
 114           else if (!(MI.getDesc().TSFlags & SystemZII::Has20BitOffset))
 115             SortingObjects[Index].D12Count++;
 116         }
 117       }
 118     }
 119
 120   // Sort all objects for short/paired displacements, which should be
 121   // sufficient as it seems like all frame objects typically are within the
 122   // long displacement range.  Sorting works by computing the "density" as
 123   // Count / ObjectSize. The comparisons of two such fractions are refactored
 124   // by multiplying both sides with A.ObjectSize * B.ObjectSize, in order to
 125   // eliminate the (fp) divisions.  A higher density object needs to go after
 126   // in the list in order for it to end up lower on the stack.
 127   auto CmpD12 = [](const SZFrameSortingObj &A, const SZFrameSortingObj &B) {
 128     // Put all invalid and variable sized objects at the end.
 129     if (!A.IsValid || !B.IsValid)
 130       return A.IsValid;
 131     if (!A.ObjectSize || !B.ObjectSize)
 132       return A.ObjectSize > 0;
 133     uint64_t ADensityCmp = A.D12Count * B.ObjectSize;
 134     uint64_t BDensityCmp = B.D12Count * A.ObjectSize;
 135     if (ADensityCmp != BDensityCmp)
 136       return ADensityCmp < BDensityCmp;
 137     return A.DPairCount * B.ObjectSize < B.DPairCount * A.ObjectSize;
 138   };
 139   std::stable_sort(SortingObjects.begin(), SortingObjects.end(), CmpD12);
 140
 141   // Now modify the original list to represent the final order that
 142   // we want.
 143   unsigned Idx = 0;
 144   for (auto &Obj : SortingObjects) {
 145     // All invalid items are sorted at the end, so it's safe to stop.
 146     if (!Obj.IsValid)
 147       break;
 148     ObjectsToAllocate[Idx++] = Obj.ObjectIndex;
 149   }
 150 }
 151
 152 bool SystemZFrameLowering::hasReservedCallFrame(
 153     const MachineFunction &MF) const {
 154   // The ELF ABI requires us to allocate 160 bytes of stack space for the
 155   // callee, with any outgoing stack arguments being placed above that. It
 156   // seems better to make that area a permanent feature of the frame even if
 157   // we're using a frame pointer. Similarly, 64-bit XPLINK requires 96 bytes
 158   // of stack space for the register save area.
 159   return true;
 160 }
 161
 162 bool SystemZELFFrameLowering::assignCalleeSavedSpillSlots(
 163     MachineFunction &MF, const TargetRegisterInfo *TRI,
 164     std::vector<CalleeSavedInfo> &CSI) const {
 165   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 166   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 167   bool IsVarArg = MF.getFunction().isVarArg();
 168   if (CSI.empty())
 169     return true; // Early exit if no callee saved registers are modified!
 170
 171   unsigned LowGPR = 0;
 172   unsigned HighGPR = SystemZ::R15D;
 173   int StartSPOffset = SystemZMC::ELFCallFrameSize;
 174   for (auto &CS : CSI) {
 175     Register Reg = CS.getReg();
 176     int Offset = getRegSpillOffset(MF, Reg);
 177     if (Offset) {
 178       if (SystemZ::GR64BitRegClass.contains(Reg) && StartSPOffset > Offset) {
 179         LowGPR = Reg;
 180         StartSPOffset = Offset;
 181       }
 182       Offset -= SystemZMC::ELFCallFrameSize;
 183       int FrameIdx =
 184           MFFrame.CreateFixedSpillStackObject(getPointerSize(), Offset);
 185       CS.setFrameIdx(FrameIdx);
 186     } else
 187       CS.setFrameIdx(INT32_MAX);
 188   }
 189
 190   // Save the range of call-saved registers, for use by the
 191   // prologue/epilogue inserters.
 192   ZFI->setRestoreGPRRegs(LowGPR, HighGPR, StartSPOffset);
 193   if (IsVarArg) {
 194     // Also save the GPR varargs, if any.  R6D is call-saved, so would
 195     // already be included, but we also need to handle the call-clobbered
 196     // argument registers.
 197     Register FirstGPR = ZFI->getVarArgsFirstGPR();
 198     if (FirstGPR < SystemZ::ELFNumArgGPRs) {
 199       unsigned Reg = SystemZ::ELFArgGPRs[FirstGPR];
 200       int Offset = getRegSpillOffset(MF, Reg);
 201       if (StartSPOffset > Offset) {
 202         LowGPR = Reg; StartSPOffset = Offset;
 203       }
 204     }
 205   }
 206   ZFI->setSpillGPRRegs(LowGPR, HighGPR, StartSPOffset);
 207
 208   // Create fixed stack objects for the remaining registers.
 209   int CurrOffset = -SystemZMC::ELFCallFrameSize;
 210   if (usePackedStack(MF))
 211     CurrOffset += StartSPOffset;
 212
 213   for (auto &CS : CSI) {
 214     if (CS.getFrameIdx() != INT32_MAX)
 215       continue;
 216     Register Reg = CS.getReg();
 217     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
 218     unsigned Size = TRI->getSpillSize(*RC);
 219     CurrOffset -= Size;
 220     assert(CurrOffset % 8 == 0 &&
 221            "8-byte alignment required for for all register save slots");
 222     int FrameIdx = MFFrame.CreateFixedSpillStackObject(Size, CurrOffset);
 223     CS.setFrameIdx(FrameIdx);
 224   }
 225
 226   return true;
 227 }
 228
 229 void SystemZELFFrameLowering::determineCalleeSaves(MachineFunction &MF,
 230                                                    BitVector &SavedRegs,
 231                                                    RegScavenger *RS) const {
 232   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
 233
 234   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 235   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
 236   bool HasFP = hasFP(MF);
 237   SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
 238   bool IsVarArg = MF.getFunction().isVarArg();
 239
 240   // va_start stores incoming FPR varargs in the normal way, but delegates
 241   // the saving of incoming GPR varargs to spillCalleeSavedRegisters().
 242   // Record these pending uses, which typically include the call-saved
 243   // argument register R6D.
 244   if (IsVarArg)
 245     for (unsigned I = MFI->getVarArgsFirstGPR(); I < SystemZ::ELFNumArgGPRs; ++I)
 246       SavedRegs.set(SystemZ::ELFArgGPRs[I]);
 247
 248   // If there are any landing pads, entering them will modify r6/r7.
 249   if (!MF.getLandingPads().empty()) {
 250     SavedRegs.set(SystemZ::R6D);
 251     SavedRegs.set(SystemZ::R7D);
 252   }
 253
 254   // If the function requires a frame pointer, record that the hard
 255   // frame pointer will be clobbered.
 256   if (HasFP)
 257     SavedRegs.set(SystemZ::R11D);
 258
 259   // If the function calls other functions, record that the return
 260   // address register will be clobbered.
 261   if (MFFrame.hasCalls())
 262     SavedRegs.set(SystemZ::R14D);
 263
 264   // If we are saving GPRs other than the stack pointer, we might as well
 265   // save and restore the stack pointer at the same time, via STMG and LMG.
 266   // This allows the deallocation to be done by the LMG, rather than needing
 267   // a separate %r15 addition.
 268   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
 269   for (unsigned I = 0; CSRegs[I]; ++I) {
 270     unsigned Reg = CSRegs[I];
 271     if (SystemZ::GR64BitRegClass.contains(Reg) && SavedRegs.test(Reg)) {
 272       SavedRegs.set(SystemZ::R15D);
 273       break;
 274     }
 275   }
 276 }
 277
 278 SystemZELFFrameLowering::SystemZELFFrameLowering(unsigned PointerSize)
 279     : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(8), 0,
 280                            Align(8), /* StackRealignable */ false, PointerSize),
 281       RegSpillOffsets(0) {
 282
 283   // Due to the SystemZ ABI, the DWARF CFA (Canonical Frame Address) is not
 284   // equal to the incoming stack pointer, but to incoming stack pointer plus
 285   // 160.  Instead of using a Local Area Offset, the Register save area will
 286   // be occupied by fixed frame objects, and all offsets are actually
 287   // relative to CFA.
 288
 289   // Create a mapping from register number to save slot offset.
 290   // These offsets are relative to the start of the register save area.
 291   RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS);
 292   for (const auto &Entry : ELFSpillOffsetTable)
 293     RegSpillOffsets[Entry.Reg] = Entry.Offset;
 294 }
 295
 296 // Add GPR64 to the save instruction being built by MIB, which is in basic
 297 // block MBB.  IsImplicit says whether this is an explicit operand to the
 298 // instruction, or an implicit one that comes between the explicit start
 299 // and end registers.
 300 static void addSavedGPR(MachineBasicBlock &MBB, MachineInstrBuilder &MIB,
 301                         unsigned GPR64, bool IsImplicit) {
 302   const TargetRegisterInfo *RI =
 303       MBB.getParent()->getSubtarget().getRegisterInfo();
 304   Register GPR32 = RI->getSubReg(GPR64, SystemZ::subreg_l32);
 305   bool IsLive = MBB.isLiveIn(GPR64) || MBB.isLiveIn(GPR32);
 306   if (!IsLive || !IsImplicit) {
 307     MIB.addReg(GPR64, getImplRegState(IsImplicit) | getKillRegState(!IsLive));
 308     if (!IsLive)
 309       MBB.addLiveIn(GPR64);
 310   }
 311 }
 312
 313 bool SystemZELFFrameLowering::spillCalleeSavedRegisters(
 314     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
 315     ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
 316   if (CSI.empty())
 317     return false;
 318
 319   MachineFunction &MF = *MBB.getParent();
 320   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 321   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 322   bool IsVarArg = MF.getFunction().isVarArg();
 323   DebugLoc DL;
 324
 325   // Save GPRs
 326   SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs();
 327   if (SpillGPRs.LowGPR) {
 328     assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR &&
 329            "Should be saving %r15 and something else");
 330
 331     // Build an STMG instruction.
 332     MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STMG));
 333
 334     // Add the explicit register operands.
 335     addSavedGPR(MBB, MIB, SpillGPRs.LowGPR, false);
 336     addSavedGPR(MBB, MIB, SpillGPRs.HighGPR, false);
 337
 338     // Add the address.
 339     MIB.addReg(SystemZ::R15D).addImm(SpillGPRs.GPROffset);
 340
 341     // Make sure all call-saved GPRs are included as operands and are
 342     // marked as live on entry.
 343     for (const CalleeSavedInfo &I : CSI) {
 344       Register Reg = I.getReg();
 345       if (SystemZ::GR64BitRegClass.contains(Reg))
 346         addSavedGPR(MBB, MIB, Reg, true);
 347     }
 348
 349     // ...likewise GPR varargs.
 350     if (IsVarArg)
 351       for (unsigned I = ZFI->getVarArgsFirstGPR(); I < SystemZ::ELFNumArgGPRs; ++I)
 352         addSavedGPR(MBB, MIB, SystemZ::ELFArgGPRs[I], true);
 353   }
 354
 355   // Save FPRs/VRs in the normal TargetInstrInfo way.
 356   for (const CalleeSavedInfo &I : CSI) {
 357     Register Reg = I.getReg();
 358     if (SystemZ::FP64BitRegClass.contains(Reg)) {
 359       MBB.addLiveIn(Reg);
 360       TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(),
 361                                &SystemZ::FP64BitRegClass, TRI, Register());
 362     }
 363     if (SystemZ::VR128BitRegClass.contains(Reg)) {
 364       MBB.addLiveIn(Reg);
 365       TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(),
 366                                &SystemZ::VR128BitRegClass, TRI, Register());
 367     }
 368   }
 369
 370   return true;
 371 }
 372
 373 bool SystemZELFFrameLowering::restoreCalleeSavedRegisters(
 374     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
 375     MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
 376   if (CSI.empty())
 377     return false;
 378
 379   MachineFunction &MF = *MBB.getParent();
 380   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 381   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 382   bool HasFP = hasFP(MF);
 383   DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 384
 385   // Restore FPRs/VRs in the normal TargetInstrInfo way.
 386   for (const CalleeSavedInfo &I : CSI) {
 387     Register Reg = I.getReg();
 388     if (SystemZ::FP64BitRegClass.contains(Reg))
 389       TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(),
 390                                 &SystemZ::FP64BitRegClass, TRI, Register());
 391     if (SystemZ::VR128BitRegClass.contains(Reg))
 392       TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(),
 393                                 &SystemZ::VR128BitRegClass, TRI, Register());
 394   }
 395
 396   // Restore call-saved GPRs (but not call-clobbered varargs, which at
 397   // this point might hold return values).
 398   SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
 399   if (RestoreGPRs.LowGPR) {
 400     // If we saved any of %r2-%r5 as varargs, we should also be saving
 401     // and restoring %r6.  If we're saving %r6 or above, we should be
 402     // restoring it too.
 403     assert(RestoreGPRs.LowGPR != RestoreGPRs.HighGPR &&
 404            "Should be loading %r15 and something else");
 405
 406     // Build an LMG instruction.
 407     MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG));
 408
 409     // Add the explicit register operands.
 410     MIB.addReg(RestoreGPRs.LowGPR, RegState::Define);
 411     MIB.addReg(RestoreGPRs.HighGPR, RegState::Define);
 412
 413     // Add the address.
 414     MIB.addReg(HasFP ? SystemZ::R11D : SystemZ::R15D);
 415     MIB.addImm(RestoreGPRs.GPROffset);
 416
 417     // Do a second scan adding regs as being defined by instruction
 418     for (const CalleeSavedInfo &I : CSI) {
 419       Register Reg = I.getReg();
 420       if (Reg != RestoreGPRs.LowGPR && Reg != RestoreGPRs.HighGPR &&
 421           SystemZ::GR64BitRegClass.contains(Reg))
 422         MIB.addReg(Reg, RegState::ImplicitDefine);
 423     }
 424   }
 425
 426   return true;
 427 }
 428
 429 void SystemZELFFrameLowering::processFunctionBeforeFrameFinalized(
 430     MachineFunction &MF, RegScavenger *RS) const {
 431   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 432   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 433   MachineRegisterInfo *MRI = &MF.getRegInfo();
 434   bool BackChain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
 435
 436   if (!usePackedStack(MF) || BackChain)
 437     // Create the incoming register save area.
 438     getOrCreateFramePointerSaveIndex(MF);
 439
 440   // Get the size of our stack frame to be allocated ...
 441   uint64_t StackSize = (MFFrame.estimateStackSize(MF) +
 442                         SystemZMC::ELFCallFrameSize);
 443   // ... and the maximum offset we may need to reach into the
 444   // caller's frame to access the save area or stack arguments.
 445   int64_t MaxArgOffset = 0;
 446   for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I)
 447     if (MFFrame.getObjectOffset(I) >= 0) {
 448       int64_t ArgOffset = MFFrame.getObjectOffset(I) +
 449                           MFFrame.getObjectSize(I);
 450       MaxArgOffset = std::max(MaxArgOffset, ArgOffset);
 451     }
 452
 453   uint64_t MaxReach = StackSize + MaxArgOffset;
 454   if (!isUInt<12>(MaxReach)) {
 455     // We may need register scavenging slots if some parts of the frame
 456     // are outside the reach of an unsigned 12-bit displacement.
 457     // Create 2 for the case where both addresses in an MVC are
 458     // out of range.
 459     RS->addScavengingFrameIndex(
 460         MFFrame.CreateSpillStackObject(getPointerSize(), Align(8)));
 461     RS->addScavengingFrameIndex(
 462         MFFrame.CreateSpillStackObject(getPointerSize(), Align(8)));
 463   }
 464
 465   // If R6 is used as an argument register it is still callee saved. If it in
 466   // this case is not clobbered (and restored) it should never be marked as
 467   // killed.
 468   if (MF.front().isLiveIn(SystemZ::R6D) &&
 469       ZFI->getRestoreGPRRegs().LowGPR != SystemZ::R6D)
 470     for (auto &MO : MRI->use_nodbg_operands(SystemZ::R6D))
 471       MO.setIsKill(false);
 472 }
 473
 474 // Emit instructions before MBBI (in MBB) to add NumBytes to Reg.
 475 static void emitIncrement(MachineBasicBlock &MBB,
 476                           MachineBasicBlock::iterator &MBBI, const DebugLoc &DL,
 477                           Register Reg, int64_t NumBytes,
 478                           const TargetInstrInfo *TII) {
 479   while (NumBytes) {
 480     unsigned Opcode;
 481     int64_t ThisVal = NumBytes;
 482     if (isInt<16>(NumBytes))
 483       Opcode = SystemZ::AGHI;
 484     else {
 485       Opcode = SystemZ::AGFI;
 486       // Make sure we maintain 8-byte stack alignment.
 487       int64_t MinVal = -uint64_t(1) << 31;
 488       int64_t MaxVal = (int64_t(1) << 31) - 8;
 489       if (ThisVal < MinVal)
 490         ThisVal = MinVal;
 491       else if (ThisVal > MaxVal)
 492         ThisVal = MaxVal;
 493     }
 494     MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII->get(Opcode), Reg)
 495       .addReg(Reg).addImm(ThisVal);
 496     // The CC implicit def is dead.
 497     MI->getOperand(3).setIsDead();
 498     NumBytes -= ThisVal;
 499   }
 500 }
 501
 502 // Add CFI for the new CFA offset.
 503 static void buildCFAOffs(MachineBasicBlock &MBB,
 504                          MachineBasicBlock::iterator MBBI,
 505                          const DebugLoc &DL, int Offset,
 506                          const SystemZInstrInfo *ZII) {
 507   unsigned CFIIndex = MBB.getParent()->addFrameInst(
 508     MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset));
 509   BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
 510     .addCFIIndex(CFIIndex);
 511 }
 512
 513 // Add CFI for the new frame location.
 514 static void buildDefCFAReg(MachineBasicBlock &MBB,
 515                            MachineBasicBlock::iterator MBBI,
 516                            const DebugLoc &DL, unsigned Reg,
 517                            const SystemZInstrInfo *ZII) {
 518   MachineFunction &MF = *MBB.getParent();
 519   const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 520   unsigned RegNum = MRI->getDwarfRegNum(Reg, true);
 521   unsigned CFIIndex = MF.addFrameInst(
 522                         MCCFIInstruction::createDefCfaRegister(nullptr, RegNum));
 523   BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
 524     .addCFIIndex(CFIIndex);
 525 }
 526
 527 void SystemZELFFrameLowering::emitPrologue(MachineFunction &MF,
 528                                            MachineBasicBlock &MBB) const {
 529   assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
 530   const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
 531   const SystemZTargetLowering &TLI = *STI.getTargetLowering();
 532   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 533   auto *ZII = static_cast<const SystemZInstrInfo *>(STI.getInstrInfo());
 534   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 535   MachineBasicBlock::iterator MBBI = MBB.begin();
 536   const MCRegisterInfo *MRI = MF.getContext().getRegisterInfo();
 537   const std::vector<CalleeSavedInfo> &CSI = MFFrame.getCalleeSavedInfo();
 538   bool HasFP = hasFP(MF);
 539
 540   // In GHC calling convention C stack space, including the ABI-defined
 541   // 160-byte base area, is (de)allocated by GHC itself.  This stack space may
 542   // be used by LLVM as spill slots for the tail recursive GHC functions.  Thus
 543   // do not allocate stack space here, too.
 544   if (MF.getFunction().getCallingConv() == CallingConv::GHC) {
 545     if (MFFrame.getStackSize() > 2048 * sizeof(long)) {
 546       report_fatal_error(
 547           "Pre allocated stack space for GHC function is too small");
 548     }
 549     if (HasFP) {
 550       report_fatal_error(
 551           "In GHC calling convention a frame pointer is not supported");
 552     }
 553     MFFrame.setStackSize(MFFrame.getStackSize() + SystemZMC::ELFCallFrameSize);
 554     return;
 555   }
 556
 557   // Debug location must be unknown since the first debug location is used
 558   // to determine the end of the prologue.
 559   DebugLoc DL;
 560
 561   // The current offset of the stack pointer from the CFA.
 562   int64_t SPOffsetFromCFA = -SystemZMC::ELFCFAOffsetFromInitialSP;
 563
 564   if (ZFI->getSpillGPRRegs().LowGPR) {
 565     // Skip over the GPR saves.
 566     if (MBBI != MBB.end() && MBBI->getOpcode() == SystemZ::STMG)
 567       ++MBBI;
 568     else
 569       llvm_unreachable("Couldn't skip over GPR saves");
 570
 571     // Add CFI for the GPR saves.
 572     for (auto &Save : CSI) {
 573       Register Reg = Save.getReg();
 574       if (SystemZ::GR64BitRegClass.contains(Reg)) {
 575         int FI = Save.getFrameIdx();
 576         int64_t Offset = MFFrame.getObjectOffset(FI);
 577         unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
 578             nullptr, MRI->getDwarfRegNum(Reg, true), Offset));
 579         BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
 580             .addCFIIndex(CFIIndex);
 581       }
 582     }
 583   }
 584
 585   uint64_t StackSize = MFFrame.getStackSize();
 586   // We need to allocate the ABI-defined 160-byte base area whenever
 587   // we allocate stack space for our own use and whenever we call another
 588   // function.
 589   bool HasStackObject = false;
 590   for (unsigned i = 0, e = MFFrame.getObjectIndexEnd(); i != e; ++i)
 591     if (!MFFrame.isDeadObjectIndex(i)) {
 592       HasStackObject = true;
 593       break;
 594     }
 595   if (HasStackObject || MFFrame.hasCalls())
 596     StackSize += SystemZMC::ELFCallFrameSize;
 597   // Don't allocate the incoming reg save area.
 598   StackSize = StackSize > SystemZMC::ELFCallFrameSize
 599                   ? StackSize - SystemZMC::ELFCallFrameSize
 600                   : 0;
 601   MFFrame.setStackSize(StackSize);
 602
 603   if (StackSize) {
 604     // Allocate StackSize bytes.
 605     int64_t Delta = -int64_t(StackSize);
 606     const unsigned ProbeSize = TLI.getStackProbeSize(MF);
 607     bool FreeProbe = (ZFI->getSpillGPRRegs().GPROffset &&
 608            (ZFI->getSpillGPRRegs().GPROffset + StackSize) < ProbeSize);
 609     if (!FreeProbe &&
 610         MF.getSubtarget().getTargetLowering()->hasInlineStackProbe(MF)) {
 611       // Stack probing may involve looping, but splitting the prologue block
 612       // is not possible at this point since it would invalidate the
 613       // SaveBlocks / RestoreBlocks sets of PEI in the single block function
 614       // case. Build a pseudo to be handled later by inlineStackProbe().
 615       BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::PROBED_STACKALLOC))
 616         .addImm(StackSize);
 617     }
 618     else {
 619       bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
 620       // If we need backchain, save current stack pointer.  R1 is free at
 621       // this point.
 622       if (StoreBackchain)
 623         BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR))
 624           .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
 625       emitIncrement(MBB, MBBI, DL, SystemZ::R15D, Delta, ZII);
 626       buildCFAOffs(MBB, MBBI, DL, SPOffsetFromCFA + Delta, ZII);
 627       if (StoreBackchain)
 628         BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
 629           .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
 630           .addImm(getBackchainOffset(MF)).addReg(0);
 631     }
 632     SPOffsetFromCFA += Delta;
 633   }
 634
 635   if (HasFP) {
 636     // Copy the base of the frame to R11.
 637     BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R11D)
 638       .addReg(SystemZ::R15D);
 639
 640     // Add CFI for the new frame location.
 641     buildDefCFAReg(MBB, MBBI, DL, SystemZ::R11D, ZII);
 642
 643     // Mark the FramePtr as live at the beginning of every block except
 644     // the entry block.  (We'll have marked R11 as live on entry when
 645     // saving the GPRs.)
 646     for (MachineBasicBlock &MBBJ : llvm::drop_begin(MF))
 647       MBBJ.addLiveIn(SystemZ::R11D);
 648   }
 649
 650   // Skip over the FPR/VR saves.
 651   SmallVector<unsigned, 8> CFIIndexes;
 652   for (auto &Save : CSI) {
 653     Register Reg = Save.getReg();
 654     if (SystemZ::FP64BitRegClass.contains(Reg)) {
 655       if (MBBI != MBB.end() &&
 656           (MBBI->getOpcode() == SystemZ::STD ||
 657            MBBI->getOpcode() == SystemZ::STDY))
 658         ++MBBI;
 659       else
 660         llvm_unreachable("Couldn't skip over FPR save");
 661     } else if (SystemZ::VR128BitRegClass.contains(Reg)) {
 662       if (MBBI != MBB.end() &&
 663           MBBI->getOpcode() == SystemZ::VST)
 664         ++MBBI;
 665       else
 666         llvm_unreachable("Couldn't skip over VR save");
 667     } else
 668       continue;
 669
 670     // Add CFI for the this save.
 671     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
 672     Register IgnoredFrameReg;
 673     int64_t Offset =
 674         getFrameIndexReference(MF, Save.getFrameIdx(), IgnoredFrameReg)
 675             .getFixed();
 676
 677     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
 678           nullptr, DwarfReg, SPOffsetFromCFA + Offset));
 679     CFIIndexes.push_back(CFIIndex);
 680   }
 681   // Complete the CFI for the FPR/VR saves, modelling them as taking effect
 682   // after the last save.
 683   for (auto CFIIndex : CFIIndexes) {
 684     BuildMI(MBB, MBBI, DL, ZII->get(TargetOpcode::CFI_INSTRUCTION))
 685         .addCFIIndex(CFIIndex);
 686   }
 687 }
 688
 689 void SystemZELFFrameLowering::emitEpilogue(MachineFunction &MF,
 690                                            MachineBasicBlock &MBB) const {
 691   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
 692   auto *ZII =
 693       static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
 694   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 695   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 696
 697   // See SystemZELFFrameLowering::emitPrologue
 698   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
 699     return;
 700
 701   // Skip the return instruction.
 702   assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
 703
 704   uint64_t StackSize = MFFrame.getStackSize();
 705   if (ZFI->getRestoreGPRRegs().LowGPR) {
 706     --MBBI;
 707     unsigned Opcode = MBBI->getOpcode();
 708     if (Opcode != SystemZ::LMG)
 709       llvm_unreachable("Expected to see callee-save register restore code");
 710
 711     unsigned AddrOpNo = 2;
 712     DebugLoc DL = MBBI->getDebugLoc();
 713     uint64_t Offset = StackSize + MBBI->getOperand(AddrOpNo + 1).getImm();
 714     unsigned NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
 715
 716     // If the offset is too large, use the largest stack-aligned offset
 717     // and add the rest to the base register (the stack or frame pointer).
 718     if (!NewOpcode) {
 719       uint64_t NumBytes = Offset - 0x7fff8;
 720       emitIncrement(MBB, MBBI, DL, MBBI->getOperand(AddrOpNo).getReg(),
 721                     NumBytes, ZII);
 722       Offset -= NumBytes;
 723       NewOpcode = ZII->getOpcodeForOffset(Opcode, Offset);
 724       assert(NewOpcode && "No restore instruction available");
 725     }
 726
 727     MBBI->setDesc(ZII->get(NewOpcode));
 728     MBBI->getOperand(AddrOpNo + 1).ChangeToImmediate(Offset);
 729   } else if (StackSize) {
 730     DebugLoc DL = MBBI->getDebugLoc();
 731     emitIncrement(MBB, MBBI, DL, SystemZ::R15D, StackSize, ZII);
 732   }
 733 }
 734
 735 void SystemZELFFrameLowering::inlineStackProbe(
 736     MachineFunction &MF, MachineBasicBlock &PrologMBB) const {
 737   auto *ZII =
 738     static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
 739   const SystemZSubtarget &STI = MF.getSubtarget<SystemZSubtarget>();
 740   const SystemZTargetLowering &TLI = *STI.getTargetLowering();
 741
 742   MachineInstr *StackAllocMI = nullptr;
 743   for (MachineInstr &MI : PrologMBB)
 744     if (MI.getOpcode() == SystemZ::PROBED_STACKALLOC) {
 745       StackAllocMI = &MI;
 746       break;
 747     }
 748   if (StackAllocMI == nullptr)
 749     return;
 750   uint64_t StackSize = StackAllocMI->getOperand(0).getImm();
 751   const unsigned ProbeSize = TLI.getStackProbeSize(MF);
 752   uint64_t NumFullBlocks = StackSize / ProbeSize;
 753   uint64_t Residual = StackSize % ProbeSize;
 754   int64_t SPOffsetFromCFA = -SystemZMC::ELFCFAOffsetFromInitialSP;
 755   MachineBasicBlock *MBB = &PrologMBB;
 756   MachineBasicBlock::iterator MBBI = StackAllocMI;
 757   const DebugLoc DL = StackAllocMI->getDebugLoc();
 758
 759   // Allocate a block of Size bytes on the stack and probe it.
 760   auto allocateAndProbe = [&](MachineBasicBlock &InsMBB,
 761                               MachineBasicBlock::iterator InsPt, unsigned Size,
 762                               bool EmitCFI) -> void {
 763     emitIncrement(InsMBB, InsPt, DL, SystemZ::R15D, -int64_t(Size), ZII);
 764     if (EmitCFI) {
 765       SPOffsetFromCFA -= Size;
 766       buildCFAOffs(InsMBB, InsPt, DL, SPOffsetFromCFA, ZII);
 767     }
 768     // Probe by means of a volatile compare.
 769     MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo(),
 770       MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, 8, Align(1));
 771     BuildMI(InsMBB, InsPt, DL, ZII->get(SystemZ::CG))
 772       .addReg(SystemZ::R0D, RegState::Undef)
 773       .addReg(SystemZ::R15D).addImm(Size - 8).addReg(0)
 774       .addMemOperand(MMO);
 775   };
 776
 777   bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
 778   if (StoreBackchain)
 779     BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR))
 780       .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
 781
 782   MachineBasicBlock *DoneMBB = nullptr;
 783   MachineBasicBlock *LoopMBB = nullptr;
 784   if (NumFullBlocks < 3) {
 785     // Emit unrolled probe statements.
 786     for (unsigned int i = 0; i < NumFullBlocks; i++)
 787       allocateAndProbe(*MBB, MBBI, ProbeSize, true/*EmitCFI*/);
 788   } else {
 789     // Emit a loop probing the pages.
 790     uint64_t LoopAlloc = ProbeSize * NumFullBlocks;
 791     SPOffsetFromCFA -= LoopAlloc;
 792
 793     // Use R0D to hold the exit value.
 794     BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R0D)
 795       .addReg(SystemZ::R15D);
 796     buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R0D, ZII);
 797     emitIncrement(*MBB, MBBI, DL, SystemZ::R0D, -int64_t(LoopAlloc), ZII);
 798     buildCFAOffs(*MBB, MBBI, DL, -int64_t(SystemZMC::ELFCallFrameSize + LoopAlloc),
 799                  ZII);
 800
 801     DoneMBB = SystemZ::splitBlockBefore(MBBI, MBB);
 802     LoopMBB = SystemZ::emitBlockAfter(MBB);
 803     MBB->addSuccessor(LoopMBB);
 804     LoopMBB->addSuccessor(LoopMBB);
 805     LoopMBB->addSuccessor(DoneMBB);
 806
 807     MBB = LoopMBB;
 808     allocateAndProbe(*MBB, MBB->end(), ProbeSize, false/*EmitCFI*/);
 809     BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::CLGR))
 810       .addReg(SystemZ::R15D).addReg(SystemZ::R0D);
 811     BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::BRC))
 812       .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_GT).addMBB(MBB);
 813
 814     MBB = DoneMBB;
 815     MBBI = DoneMBB->begin();
 816     buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R15D, ZII);
 817   }
 818
 819   if (Residual)
 820     allocateAndProbe(*MBB, MBBI, Residual, true/*EmitCFI*/);
 821
 822   if (StoreBackchain)
 823     BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::STG))
 824       .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
 825       .addImm(getBackchainOffset(MF)).addReg(0);
 826
 827   StackAllocMI->eraseFromParent();
 828   if (DoneMBB != nullptr) {
 829     // Compute the live-in lists for the new blocks.
 830     fullyRecomputeLiveIns({DoneMBB, LoopMBB});
 831   }
 832 }
 833
 834 bool SystemZELFFrameLowering::hasFPImpl(const MachineFunction &MF) const {
 835   return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
 836           MF.getFrameInfo().hasVarSizedObjects());
 837 }
 838
 839 StackOffset SystemZELFFrameLowering::getFrameIndexReference(
 840     const MachineFunction &MF, int FI, Register &FrameReg) const {
 841   // Our incoming SP is actually SystemZMC::ELFCallFrameSize below the CFA, so
 842   // add that difference here.
 843   StackOffset Offset =
 844       TargetFrameLowering::getFrameIndexReference(MF, FI, FrameReg);
 845   return Offset + StackOffset::getFixed(SystemZMC::ELFCallFrameSize);
 846 }
 847
 848 unsigned SystemZELFFrameLowering::getRegSpillOffset(MachineFunction &MF,
 849                                                     Register Reg) const {
 850   bool IsVarArg = MF.getFunction().isVarArg();
 851   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
 852   bool BackChain = Subtarget.hasBackChain();
 853   bool SoftFloat = Subtarget.hasSoftFloat();
 854   unsigned Offset = RegSpillOffsets[Reg];
 855   if (usePackedStack(MF) && !(IsVarArg && !SoftFloat)) {
 856     if (SystemZ::GR64BitRegClass.contains(Reg))
 857       // Put all GPRs at the top of the Register save area with packed
 858       // stack. Make room for the backchain if needed.
 859       Offset += BackChain ? 24 : 32;
 860     else
 861       Offset = 0;
 862   }
 863   return Offset;
 864 }
 865
 866 int SystemZELFFrameLowering::getOrCreateFramePointerSaveIndex(
 867     MachineFunction &MF) const {
 868   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 869   int FI = ZFI->getFramePointerSaveIndex();
 870   if (!FI) {
 871     MachineFrameInfo &MFFrame = MF.getFrameInfo();
 872     int Offset = getBackchainOffset(MF) - SystemZMC::ELFCallFrameSize;
 873     FI = MFFrame.CreateFixedObject(getPointerSize(), Offset, false);
 874     ZFI->setFramePointerSaveIndex(FI);
 875   }
 876   return FI;
 877 }
 878
 879 bool SystemZELFFrameLowering::usePackedStack(MachineFunction &MF) const {
 880   bool HasPackedStackAttr = MF.getFunction().hasFnAttribute("packed-stack");
 881   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
 882   bool BackChain = Subtarget.hasBackChain();
 883   bool SoftFloat = Subtarget.hasSoftFloat();
 884   if (HasPackedStackAttr && BackChain && !SoftFloat)
 885     report_fatal_error("packed-stack + backchain + hard-float is unsupported.");
 886   bool CallConv = MF.getFunction().getCallingConv() != CallingConv::GHC;
 887   return HasPackedStackAttr && CallConv;
 888 }
 889
 890 SystemZXPLINKFrameLowering::SystemZXPLINKFrameLowering(unsigned PointerSize)
 891     : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(32), 0,
 892                            Align(32), /* StackRealignable */ false,
 893                            PointerSize),
 894       RegSpillOffsets(-1) {
 895
 896   // Create a mapping from register number to save slot offset.
 897   // These offsets are relative to the start of the local are area.
 898   RegSpillOffsets.grow(SystemZ::NUM_TARGET_REGS);
 899   for (const auto &Entry : XPLINKSpillOffsetTable)
 900     RegSpillOffsets[Entry.Reg] = Entry.Offset;
 901 }
 902
 903 int SystemZXPLINKFrameLowering::getOrCreateFramePointerSaveIndex(
 904     MachineFunction &MF) const {
 905   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
 906   int FI = ZFI->getFramePointerSaveIndex();
 907   if (!FI) {
 908     MachineFrameInfo &MFFrame = MF.getFrameInfo();
 909     FI = MFFrame.CreateFixedObject(getPointerSize(), 0, false);
 910     MFFrame.setStackID(FI, TargetStackID::NoAlloc);
 911     ZFI->setFramePointerSaveIndex(FI);
 912   }
 913   return FI;
 914 }
 915
 916 // Checks if the function is a potential candidate for being a XPLeaf routine.
 917 static bool isXPLeafCandidate(const MachineFunction &MF) {
 918   const MachineFrameInfo &MFFrame = MF.getFrameInfo();
 919   const MachineRegisterInfo &MRI = MF.getRegInfo();
 920   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
 921   auto *Regs =
 922       static_cast<SystemZXPLINK64Registers *>(Subtarget.getSpecialRegisters());
 923
 924   // If function calls other functions including alloca, then it is not a XPLeaf
 925   // routine.
 926   if (MFFrame.hasCalls())
 927     return false;
 928
 929   // If the function has var Sized Objects, then it is not a XPLeaf routine.
 930   if (MFFrame.hasVarSizedObjects())
 931     return false;
 932
 933   // If the function adjusts the stack, then it is not a XPLeaf routine.
 934   if (MFFrame.adjustsStack())
 935     return false;
 936
 937   // If function modifies the stack pointer register, then it is not a XPLeaf
 938   // routine.
 939   if (MRI.isPhysRegModified(Regs->getStackPointerRegister()))
 940     return false;
 941
 942   // If function modifies the ADA register, then it is not a XPLeaf routine.
 943   if (MRI.isPhysRegModified(Regs->getAddressOfCalleeRegister()))
 944     return false;
 945
 946   // If function modifies the return address register, then it is not a XPLeaf
 947   // routine.
 948   if (MRI.isPhysRegModified(Regs->getReturnFunctionAddressRegister()))
 949     return false;
 950
 951   // If the backchain pointer should be stored, then it is not a XPLeaf routine.
 952   if (MF.getSubtarget<SystemZSubtarget>().hasBackChain())
 953     return false;
 954
 955   // If function acquires its own stack frame, then it is not a XPLeaf routine.
 956   // At the time this function is called, only slots for local variables are
 957   // allocated, so this is a very rough estimate.
 958   if (MFFrame.estimateStackSize(MF) > 0)
 959     return false;
 960
 961   return true;
 962 }
 963
 964 bool SystemZXPLINKFrameLowering::assignCalleeSavedSpillSlots(
 965     MachineFunction &MF, const TargetRegisterInfo *TRI,
 966     std::vector<CalleeSavedInfo> &CSI) const {
 967   MachineFrameInfo &MFFrame = MF.getFrameInfo();
 968   SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
 969   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
 970   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
 971   auto &GRRegClass = SystemZ::GR64BitRegClass;
 972
 973   // At this point, the result of isXPLeafCandidate() is not accurate because
 974   // the size of the save area has not yet been determined. If
 975   // isXPLeafCandidate() indicates a potential leaf function, and there are no
 976   // callee-save registers, then it is indeed a leaf function, and we can early
 977   // exit.
 978   // TODO: It is possible for leaf functions to use callee-saved registers.
 979   // It can use the 0-2k range between R4 and the caller's stack frame without
 980   // acquiring its own stack frame.
 981   bool IsLeaf = CSI.empty() && isXPLeafCandidate(MF);
 982   if (IsLeaf)
 983     return true;
 984
 985   // For non-leaf functions:
 986   // - the address of callee (entry point) register R6 must be saved
 987   CSI.push_back(CalleeSavedInfo(Regs.getAddressOfCalleeRegister()));
 988   CSI.back().setRestored(false);
 989
 990   // The return address register R7 must be saved and restored.
 991   CSI.push_back(CalleeSavedInfo(Regs.getReturnFunctionAddressRegister()));
 992
 993   // If the function needs a frame pointer, or if the backchain pointer should
 994   // be stored, then save the stack pointer register R4.
 995   if (hasFP(MF) || Subtarget.hasBackChain())
 996     CSI.push_back(CalleeSavedInfo(Regs.getStackPointerRegister()));
 997
 998   // If this function has an associated personality function then the
 999   // environment register R5 must be saved in the DSA.
1000   if (!MF.getLandingPads().empty())
1001     CSI.push_back(CalleeSavedInfo(Regs.getADARegister()));
1002
1003   // Scan the call-saved GPRs and find the bounds of the register spill area.
1004   Register LowRestoreGPR = 0;
1005   int LowRestoreOffset = INT32_MAX;
1006   Register LowSpillGPR = 0;
1007   int LowSpillOffset = INT32_MAX;
1008   Register HighGPR = 0;
1009   int HighOffset = -1;
1010
1011   // Query index of the saved frame pointer.
1012   int FPSI = MFI->getFramePointerSaveIndex();
1013
1014   for (auto &CS : CSI) {
1015     Register Reg = CS.getReg();
1016     int Offset = RegSpillOffsets[Reg];
1017     if (Offset >= 0) {
1018       if (GRRegClass.contains(Reg)) {
1019         if (LowSpillOffset > Offset) {
1020           LowSpillOffset = Offset;
1021           LowSpillGPR = Reg;
1022         }
1023         if (CS.isRestored() && LowRestoreOffset > Offset) {
1024           LowRestoreOffset = Offset;
1025           LowRestoreGPR = Reg;
1026         }
1027
1028         if (Offset > HighOffset) {
1029           HighOffset = Offset;
1030           HighGPR = Reg;
1031         }
1032         // Non-volatile GPRs are saved in the dedicated register save area at
1033         // the bottom of the stack and are not truly part of the "normal" stack
1034         // frame. Mark the frame index as NoAlloc to indicate it as such.
1035         unsigned RegSize = getPointerSize();
1036         int FrameIdx =
1037             (FPSI && Offset == 0)
1038                 ? FPSI
1039                 : MFFrame.CreateFixedSpillStackObject(RegSize, Offset);
1040         CS.setFrameIdx(FrameIdx);
1041         MFFrame.setStackID(FrameIdx, TargetStackID::NoAlloc);
1042       }
1043     } else {
1044       Register Reg = CS.getReg();
1045       const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1046       Align Alignment = TRI->getSpillAlign(*RC);
1047       unsigned Size = TRI->getSpillSize(*RC);
1048       Alignment = std::min(Alignment, getStackAlign());
1049       int FrameIdx = MFFrame.CreateStackObject(Size, Alignment, true);
1050       CS.setFrameIdx(FrameIdx);
1051     }
1052   }
1053
1054   // Save the range of call-saved registers, for use by the
1055   // prologue/epilogue inserters.
1056   if (LowRestoreGPR)
1057     MFI->setRestoreGPRRegs(LowRestoreGPR, HighGPR, LowRestoreOffset);
1058
1059   // Save the range of call-saved registers, for use by the epilogue inserter.
1060   assert(LowSpillGPR && "Expected registers to spill");
1061   MFI->setSpillGPRRegs(LowSpillGPR, HighGPR, LowSpillOffset);
1062
1063   return true;
1064 }
1065
1066 void SystemZXPLINKFrameLowering::determineCalleeSaves(MachineFunction &MF,
1067                                                       BitVector &SavedRegs,
1068                                                       RegScavenger *RS) const {
1069   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
1070
1071   bool HasFP = hasFP(MF);
1072   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1073   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1074
1075   // If the function requires a frame pointer, record that the hard
1076   // frame pointer will be clobbered.
1077   if (HasFP)
1078     SavedRegs.set(Regs.getFramePointerRegister());
1079 }
1080
1081 bool SystemZXPLINKFrameLowering::spillCalleeSavedRegisters(
1082     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1083     ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1084   if (CSI.empty())
1085     return true;
1086
1087   MachineFunction &MF = *MBB.getParent();
1088   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1089   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1090   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1091   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1092   SystemZ::GPRRegs SpillGPRs = ZFI->getSpillGPRRegs();
1093   DebugLoc DL;
1094
1095   // Save GPRs
1096   if (SpillGPRs.LowGPR) {
1097     assert(SpillGPRs.LowGPR != SpillGPRs.HighGPR &&
1098            "Should be saving multiple registers");
1099
1100     // Build an STM/STMG instruction.
1101     MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STMG));
1102
1103     // Add the explicit register operands.
1104     addSavedGPR(MBB, MIB, SpillGPRs.LowGPR, false);
1105     addSavedGPR(MBB, MIB, SpillGPRs.HighGPR, false);
1106
1107     // Add the address r4
1108     MIB.addReg(Regs.getStackPointerRegister());
1109
1110     // Add the partial offset
1111     // We cannot add the actual offset as, at the stack is not finalized
1112     MIB.addImm(SpillGPRs.GPROffset);
1113
1114     // Make sure all call-saved GPRs are included as operands and are
1115     // marked as live on entry.
1116     auto &GRRegClass = SystemZ::GR64BitRegClass;
1117     for (const CalleeSavedInfo &I : CSI) {
1118       Register Reg = I.getReg();
1119       if (GRRegClass.contains(Reg))
1120         addSavedGPR(MBB, MIB, Reg, true);
1121     }
1122   }
1123
1124   // Spill FPRs to the stack in the normal TargetInstrInfo way
1125   for (const CalleeSavedInfo &I : CSI) {
1126     Register Reg = I.getReg();
1127     if (SystemZ::FP64BitRegClass.contains(Reg)) {
1128       MBB.addLiveIn(Reg);
1129       TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(),
1130                                &SystemZ::FP64BitRegClass, TRI, Register());
1131     }
1132     if (SystemZ::VR128BitRegClass.contains(Reg)) {
1133       MBB.addLiveIn(Reg);
1134       TII->storeRegToStackSlot(MBB, MBBI, Reg, true, I.getFrameIdx(),
1135                                &SystemZ::VR128BitRegClass, TRI, Register());
1136     }
1137   }
1138
1139   return true;
1140 }
1141
1142 bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters(
1143     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1144     MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1145
1146   if (CSI.empty())
1147     return false;
1148
1149   MachineFunction &MF = *MBB.getParent();
1150   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1151   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1152   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1153   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1154
1155   DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
1156
1157   // Restore FPRs in the normal TargetInstrInfo way.
1158   for (const CalleeSavedInfo &I : CSI) {
1159     Register Reg = I.getReg();
1160     if (SystemZ::FP64BitRegClass.contains(Reg))
1161       TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(),
1162                                 &SystemZ::FP64BitRegClass, TRI, Register());
1163     if (SystemZ::VR128BitRegClass.contains(Reg))
1164       TII->loadRegFromStackSlot(MBB, MBBI, Reg, I.getFrameIdx(),
1165                                 &SystemZ::VR128BitRegClass, TRI, Register());
1166   }
1167
1168   // Restore call-saved GPRs (but not call-clobbered varargs, which at
1169   // this point might hold return values).
1170   SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs();
1171   if (RestoreGPRs.LowGPR) {
1172     assert(isInt<20>(Regs.getStackPointerBias() + RestoreGPRs.GPROffset));
1173     if (RestoreGPRs.LowGPR == RestoreGPRs.HighGPR)
1174       // Build an LG/L instruction.
1175       BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LG), RestoreGPRs.LowGPR)
1176           .addReg(Regs.getStackPointerRegister())
1177           .addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset)
1178           .addReg(0);
1179     else {
1180       // Build an LMG/LM instruction.
1181       MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG));
1182
1183       // Add the explicit register operands.
1184       MIB.addReg(RestoreGPRs.LowGPR, RegState::Define);
1185       MIB.addReg(RestoreGPRs.HighGPR, RegState::Define);
1186
1187       // Add the address.
1188       MIB.addReg(Regs.getStackPointerRegister());
1189       MIB.addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset);
1190
1191       // Do a second scan adding regs as being defined by instruction
1192       for (const CalleeSavedInfo &I : CSI) {
1193         Register Reg = I.getReg();
1194         if (Reg > RestoreGPRs.LowGPR && Reg < RestoreGPRs.HighGPR)
1195           MIB.addReg(Reg, RegState::ImplicitDefine);
1196       }
1197     }
1198   }
1199
1200   return true;
1201 }
1202
1203 void SystemZXPLINKFrameLowering::emitPrologue(MachineFunction &MF,
1204                                               MachineBasicBlock &MBB) const {
1205   assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
1206   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1207   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1208   MachineBasicBlock::iterator MBBI = MBB.begin();
1209   auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
1210   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1211   MachineFrameInfo &MFFrame = MF.getFrameInfo();
1212   MachineInstr *StoreInstr = nullptr;
1213
1214   determineFrameLayout(MF);
1215
1216   bool HasFP = hasFP(MF);
1217   // Debug location must be unknown since the first debug location is used
1218   // to determine the end of the prologue.
1219   DebugLoc DL;
1220   uint64_t Offset = 0;
1221
1222   const uint64_t StackSize = MFFrame.getStackSize();
1223
1224   if (ZFI->getSpillGPRRegs().LowGPR) {
1225     // Skip over the GPR saves.
1226     if ((MBBI != MBB.end()) && ((MBBI->getOpcode() == SystemZ::STMG))) {
1227       const int Operand = 3;
1228       // Now we can set the offset for the operation, since now the Stack
1229       // has been finalized.
1230       Offset = Regs.getStackPointerBias() + MBBI->getOperand(Operand).getImm();
1231       // Maximum displacement for STMG instruction.
1232       if (isInt<20>(Offset - StackSize))
1233         Offset -= StackSize;
1234       else
1235         StoreInstr = &*MBBI;
1236       MBBI->getOperand(Operand).setImm(Offset);
1237       ++MBBI;
1238     } else
1239       llvm_unreachable("Couldn't skip over GPR saves");
1240   }
1241
1242   if (StackSize) {
1243     MachineBasicBlock::iterator InsertPt = StoreInstr ? StoreInstr : MBBI;
1244     // Allocate StackSize bytes.
1245     int64_t Delta = -int64_t(StackSize);
1246
1247     // In case the STM(G) instruction also stores SP (R4), but the displacement
1248     // is too large, the SP register is manipulated first before storing,
1249     // resulting in the wrong value stored and retrieved later. In this case, we
1250     // need to temporarily save the value of SP, and store it later to memory.
1251     if (StoreInstr && HasFP) {
1252       // Insert LR r0,r4 before STMG instruction.
1253       BuildMI(MBB, InsertPt, DL, ZII->get(SystemZ::LGR))
1254           .addReg(SystemZ::R0D, RegState::Define)
1255           .addReg(SystemZ::R4D);
1256       // Insert ST r0,xxx(,r4) after STMG instruction.
1257       BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
1258           .addReg(SystemZ::R0D, RegState::Kill)
1259           .addReg(SystemZ::R4D)
1260           .addImm(Offset)
1261           .addReg(0);
1262     }
1263
1264     emitIncrement(MBB, InsertPt, DL, Regs.getStackPointerRegister(), Delta,
1265                   ZII);
1266
1267     // If the requested stack size is larger than the guard page, then we need
1268     // to check if we need to call the stack extender. This requires adding a
1269     // conditional branch, but splitting the prologue block is not possible at
1270     // this point since it would invalidate the SaveBlocks / RestoreBlocks sets
1271     // of PEI in the single block function case. Build a pseudo to be handled
1272     // later by inlineStackProbe().
1273     const uint64_t GuardPageSize = 1024 * 1024;
1274     if (StackSize > GuardPageSize) {
1275       assert(StoreInstr && "Wrong insertion point");
1276       BuildMI(MBB, InsertPt, DL, ZII->get(SystemZ::XPLINK_STACKALLOC));
1277     }
1278   }
1279
1280   if (HasFP) {
1281     // Copy the base of the frame to Frame Pointer Register.
1282     BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR),
1283             Regs.getFramePointerRegister())
1284         .addReg(Regs.getStackPointerRegister());
1285
1286     // Mark the FramePtr as live at the beginning of every block except
1287     // the entry block.  (We'll have marked R8 as live on entry when
1288     // saving the GPRs.)
1289     for (MachineBasicBlock &B : llvm::drop_begin(MF))
1290       B.addLiveIn(Regs.getFramePointerRegister());
1291   }
1292
1293   // Save GPRs used for varargs, if any.
1294   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1295   bool IsVarArg = MF.getFunction().isVarArg();
1296
1297   if (IsVarArg) {
1298     // FixedRegs is the number of used registers, accounting for shadow
1299     // registers.
1300     unsigned FixedRegs = ZFI->getVarArgsFirstGPR() + ZFI->getVarArgsFirstFPR();
1301     auto &GPRs = SystemZ::XPLINK64ArgGPRs;
1302     for (unsigned I = FixedRegs; I < SystemZ::XPLINK64NumArgGPRs; I++) {
1303       uint64_t StartOffset = MFFrame.getOffsetAdjustment() +
1304                              MFFrame.getStackSize() + Regs.getCallFrameSize() +
1305                              getOffsetOfLocalArea() + I * getPointerSize();
1306       unsigned Reg = GPRs[I];
1307       BuildMI(MBB, MBBI, DL, TII->get(SystemZ::STG))
1308           .addReg(Reg)
1309           .addReg(Regs.getStackPointerRegister())
1310           .addImm(StartOffset)
1311           .addReg(0);
1312       if (!MBB.isLiveIn(Reg))
1313         MBB.addLiveIn(Reg);
1314     }
1315   }
1316 }
1317
1318 void SystemZXPLINKFrameLowering::emitEpilogue(MachineFunction &MF,
1319                                               MachineBasicBlock &MBB) const {
1320   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1321   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1322   SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>();
1323   MachineFrameInfo &MFFrame = MF.getFrameInfo();
1324   auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
1325   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1326
1327   // Skip the return instruction.
1328   assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks");
1329
1330   uint64_t StackSize = MFFrame.getStackSize();
1331   if (StackSize) {
1332     unsigned SPReg = Regs.getStackPointerRegister();
1333     if (ZFI->getRestoreGPRRegs().LowGPR != SPReg) {
1334       DebugLoc DL = MBBI->getDebugLoc();
1335       emitIncrement(MBB, MBBI, DL, SPReg, StackSize, ZII);
1336     }
1337   }
1338 }
1339
1340 // Emit a compare of the stack pointer against the stack floor, and a call to
1341 // the LE stack extender if needed.
1342 void SystemZXPLINKFrameLowering::inlineStackProbe(
1343     MachineFunction &MF, MachineBasicBlock &PrologMBB) const {
1344   auto *ZII =
1345       static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
1346
1347   MachineInstr *StackAllocMI = nullptr;
1348   for (MachineInstr &MI : PrologMBB)
1349     if (MI.getOpcode() == SystemZ::XPLINK_STACKALLOC) {
1350       StackAllocMI = &MI;
1351       break;
1352     }
1353   if (StackAllocMI == nullptr)
1354     return;
1355
1356   bool NeedSaveSP = hasFP(MF);
1357   bool NeedSaveArg = PrologMBB.isLiveIn(SystemZ::R3D);
1358   const int64_t SaveSlotR3 = 2192;
1359
1360   MachineBasicBlock &MBB = PrologMBB;
1361   const DebugLoc DL = StackAllocMI->getDebugLoc();
1362
1363   // The 2nd half of block MBB after split.
1364   MachineBasicBlock *NextMBB;
1365
1366   // Add new basic block for the call to the stack overflow function.
1367   MachineBasicBlock *StackExtMBB =
1368       MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1369   MF.push_back(StackExtMBB);
1370
1371   // LG r3,72(,r3)
1372   BuildMI(StackExtMBB, DL, ZII->get(SystemZ::LG), SystemZ::R3D)
1373       .addReg(SystemZ::R3D)
1374       .addImm(72)
1375       .addReg(0);
1376   // BASR r3,r3
1377   BuildMI(StackExtMBB, DL, ZII->get(SystemZ::CallBASR_STACKEXT))
1378       .addReg(SystemZ::R3D);
1379   if (NeedSaveArg) {
1380     if (!NeedSaveSP) {
1381       // LGR r0,r3
1382       BuildMI(MBB, StackAllocMI, DL, ZII->get(SystemZ::LGR))
1383           .addReg(SystemZ::R0D, RegState::Define)
1384           .addReg(SystemZ::R3D);
1385     } else {
1386       // In this case, the incoming value of r4 is saved in r0 so the
1387       // latter register is unavailable. Store r3 in its corresponding
1388       // slot in the parameter list instead. Do this at the start of
1389       // the prolog before r4 is manipulated by anything else.
1390       // STG r3, 2192(r4)
1391       BuildMI(MBB, MBB.begin(), DL, ZII->get(SystemZ::STG))
1392           .addReg(SystemZ::R3D)
1393           .addReg(SystemZ::R4D)
1394           .addImm(SaveSlotR3)
1395           .addReg(0);
1396     }
1397   }
1398   // LLGT r3,1208
1399   BuildMI(MBB, StackAllocMI, DL, ZII->get(SystemZ::LLGT), SystemZ::R3D)
1400       .addReg(0)
1401       .addImm(1208)
1402       .addReg(0);
1403   // CG r4,64(,r3)
1404   BuildMI(MBB, StackAllocMI, DL, ZII->get(SystemZ::CG))
1405       .addReg(SystemZ::R4D)
1406       .addReg(SystemZ::R3D)
1407       .addImm(64)
1408       .addReg(0);
1409   // JLL b'0100',F'37'
1410   BuildMI(MBB, StackAllocMI, DL, ZII->get(SystemZ::BRC))
1411       .addImm(SystemZ::CCMASK_ICMP)
1412       .addImm(SystemZ::CCMASK_CMP_LT)
1413       .addMBB(StackExtMBB);
1414
1415   NextMBB = SystemZ::splitBlockBefore(StackAllocMI, &MBB);
1416   MBB.addSuccessor(NextMBB);
1417   MBB.addSuccessor(StackExtMBB);
1418   if (NeedSaveArg) {
1419     if (!NeedSaveSP) {
1420       // LGR r3, r0
1421       BuildMI(*NextMBB, StackAllocMI, DL, ZII->get(SystemZ::LGR))
1422           .addReg(SystemZ::R3D, RegState::Define)
1423           .addReg(SystemZ::R0D, RegState::Kill);
1424     } else {
1425       // In this case, the incoming value of r4 is saved in r0 so the
1426       // latter register is unavailable. We stored r3 in its corresponding
1427       // slot in the parameter list instead and we now restore it from there.
1428       // LGR r3, r0
1429       BuildMI(*NextMBB, StackAllocMI, DL, ZII->get(SystemZ::LGR))
1430           .addReg(SystemZ::R3D, RegState::Define)
1431           .addReg(SystemZ::R0D);
1432       // LG r3, 2192(r3)
1433       BuildMI(*NextMBB, StackAllocMI, DL, ZII->get(SystemZ::LG))
1434           .addReg(SystemZ::R3D, RegState::Define)
1435           .addReg(SystemZ::R3D)
1436           .addImm(SaveSlotR3)
1437           .addReg(0);
1438     }
1439   }
1440
1441   // Add jump back from stack extension BB.
1442   BuildMI(StackExtMBB, DL, ZII->get(SystemZ::J)).addMBB(NextMBB);
1443   StackExtMBB->addSuccessor(NextMBB);
1444
1445   StackAllocMI->eraseFromParent();
1446
1447   // Compute the live-in lists for the new blocks.
1448   fullyRecomputeLiveIns({StackExtMBB, NextMBB});
1449 }
1450
1451 bool SystemZXPLINKFrameLowering::hasFPImpl(const MachineFunction &MF) const {
1452   return (MF.getFrameInfo().hasVarSizedObjects());
1453 }
1454
1455 void SystemZXPLINKFrameLowering::processFunctionBeforeFrameFinalized(
1456     MachineFunction &MF, RegScavenger *RS) const {
1457   MachineFrameInfo &MFFrame = MF.getFrameInfo();
1458   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1459   auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
1460
1461   // Setup stack frame offset
1462   MFFrame.setOffsetAdjustment(Regs.getStackPointerBias());
1463
1464   // Nothing to do for leaf functions.
1465   uint64_t StackSize = MFFrame.estimateStackSize(MF);
1466   if (StackSize == 0 && MFFrame.getCalleeSavedInfo().empty())
1467     return;
1468
1469   // Although the XPLINK specifications for AMODE64 state that minimum size
1470   // of the param area is minimum 32 bytes and no rounding is otherwise
1471   // specified, we round this area in 64 bytes increments to be compatible
1472   // with existing compilers.
1473   MFFrame.setMaxCallFrameSize(
1474       std::max(64U, (unsigned)alignTo(MFFrame.getMaxCallFrameSize(), 64)));
1475
1476   // Add frame values with positive object offsets. Since the displacement from
1477   // the SP/FP is calculated by ObjectOffset + StackSize + Bias, object offsets
1478   // with positive values are in the caller's stack frame. We need to include
1479   // that since it is accessed by displacement to SP/FP.
1480   int64_t LargestArgOffset = 0;
1481   for (int I = MFFrame.getObjectIndexBegin(); I != 0; ++I) {
1482     if (MFFrame.getObjectOffset(I) >= 0) {
1483       int64_t ObjOffset = MFFrame.getObjectOffset(I) + MFFrame.getObjectSize(I);
1484       LargestArgOffset = std::max(ObjOffset, LargestArgOffset);
1485     }
1486   }
1487
1488   uint64_t MaxReach = (StackSize + Regs.getCallFrameSize() +
1489                        Regs.getStackPointerBias() + LargestArgOffset);
1490
1491   if (!isUInt<12>(MaxReach)) {
1492     // We may need register scavenging slots if some parts of the frame
1493     // are outside the reach of an unsigned 12-bit displacement.
1494     RS->addScavengingFrameIndex(MFFrame.CreateSpillStackObject(8, Align(8)));
1495     RS->addScavengingFrameIndex(MFFrame.CreateSpillStackObject(8, Align(8)));
1496   }
1497 }
1498
1499 // Determines the size of the frame, and creates the deferred spill objects.
1500 void SystemZXPLINKFrameLowering::determineFrameLayout(
1501     MachineFunction &MF) const {
1502   MachineFrameInfo &MFFrame = MF.getFrameInfo();
1503   const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>();
1504   auto *Regs =
1505       static_cast<SystemZXPLINK64Registers *>(Subtarget.getSpecialRegisters());
1506
1507   uint64_t StackSize = MFFrame.getStackSize();
1508   if (StackSize == 0)
1509     return;
1510
1511   // Add the size of the register save area and the reserved area to the size.
1512   StackSize += Regs->getCallFrameSize();
1513   MFFrame.setStackSize(StackSize);
1514
1515   // We now know the stack size. Update the stack objects for the register save
1516   // area now. This has no impact on the stack frame layout, as this is already
1517   // computed. However, it makes sure that all callee saved registers have a
1518   // valid offset assigned.
1519   for (int FrameIdx = MFFrame.getObjectIndexBegin(); FrameIdx != 0;
1520        ++FrameIdx) {
1521     if (MFFrame.getStackID(FrameIdx) == TargetStackID::NoAlloc) {
1522       int64_t SPOffset = MFFrame.getObjectOffset(FrameIdx);
1523       SPOffset -= StackSize;
1524       MFFrame.setObjectOffset(FrameIdx, SPOffset);
1525     }
1526   }
1527 }