llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp

   1 //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the AArch64 implementation of the TargetRegisterInfo
  10 // class.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64RegisterInfo.h"
  15 #include "AArch64FrameLowering.h"
  16 #include "AArch64InstrInfo.h"
  17 #include "AArch64MachineFunctionInfo.h"
  18 #include "AArch64Subtarget.h"
  19 #include "MCTargetDesc/AArch64AddressingModes.h"
  20 #include "MCTargetDesc/AArch64InstPrinter.h"
  21 #include "llvm/ADT/BitVector.h"
  22 #include "llvm/BinaryFormat/Dwarf.h"
  23 #include "llvm/CodeGen/MachineFrameInfo.h"
  24 #include "llvm/CodeGen/MachineInstrBuilder.h"
  25 #include "llvm/CodeGen/MachineRegisterInfo.h"
  26 #include "llvm/CodeGen/RegisterScavenging.h"
  27 #include "llvm/CodeGen/TargetFrameLowering.h"
  28 #include "llvm/IR/DebugInfoMetadata.h"
  29 #include "llvm/IR/DiagnosticInfo.h"
  30 #include "llvm/IR/Function.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include "llvm/Target/TargetOptions.h"
  33 #include "llvm/TargetParser/Triple.h"
  34
  35 using namespace llvm;
  36
  37 #define GET_CC_REGISTER_LISTS
  38 #include "AArch64GenCallingConv.inc"
  39 #define GET_REGINFO_TARGET_DESC
  40 #include "AArch64GenRegisterInfo.inc"
  41
  42 AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT)
  43     : AArch64GenRegisterInfo(AArch64::LR), TT(TT) {
  44   AArch64_MC::initLLVMToCVRegMapping(this);
  45 }
  46
  47 /// Return whether the register needs a CFI entry. Not all unwinders may know
  48 /// about SVE registers, so we assume the lowest common denominator, i.e. the
  49 /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the
  50 /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is
  51 /// returned in \p RegToUseForCFI.
  52 bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg,
  53                                       unsigned &RegToUseForCFI) const {
  54   if (AArch64::PPRRegClass.contains(Reg))
  55     return false;
  56
  57   if (AArch64::ZPRRegClass.contains(Reg)) {
  58     RegToUseForCFI = getSubReg(Reg, AArch64::dsub);
  59     for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {
  60       if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)
  61         return true;
  62     }
  63     return false;
  64   }
  65
  66   RegToUseForCFI = Reg;
  67   return true;
  68 }
  69
  70 const MCPhysReg *
  71 AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
  72   assert(MF && "Invalid MachineFunction pointer.");
  73
  74   if (MF->getFunction().getCallingConv() == CallingConv::GHC)
  75     // GHC set of callee saved regs is empty as all those regs are
  76     // used for passing STG regs around
  77     return CSR_AArch64_NoRegs_SaveList;
  78   if (MF->getFunction().getCallingConv() == CallingConv::AnyReg)
  79     return CSR_AArch64_AllRegs_SaveList;
  80
  81   if (MF->getFunction().getCallingConv() == CallingConv::ARM64EC_Thunk_X64)
  82     return CSR_Win_AArch64_Arm64EC_Thunk_SaveList;
  83
  84   // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save
  85   // lists depending on that will need to have their Darwin variant as well.
  86   if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin())
  87     return getDarwinCalleeSavedRegs(MF);
  88
  89   if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
  90     return CSR_Win_AArch64_CFGuard_Check_SaveList;
  91   if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows()) {
  92     if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
  93             ->supportSwiftError() &&
  94         MF->getFunction().getAttributes().hasAttrSomewhere(
  95             Attribute::SwiftError))
  96       return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;
  97     if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
  98       return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;
  99     return CSR_Win_AArch64_AAPCS_SaveList;
 100   }
 101   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
 102     return CSR_AArch64_AAVPCS_SaveList;
 103   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
 104     return CSR_AArch64_SVE_AAPCS_SaveList;
 105   if (MF->getFunction().getCallingConv() ==
 106           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
 107     report_fatal_error(
 108         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
 109         "only supported to improve calls to SME ACLE save/restore/disable-za "
 110         "functions, and is not intended to be used beyond that scope.");
 111   if (MF->getFunction().getCallingConv() ==
 112           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
 113     report_fatal_error(
 114         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
 115         "only supported to improve calls to SME ACLE __arm_sme_state "
 116         "and is not intended to be used beyond that scope.");
 117   if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
 118           ->supportSwiftError() &&
 119       MF->getFunction().getAttributes().hasAttrSomewhere(
 120           Attribute::SwiftError))
 121     return CSR_AArch64_AAPCS_SwiftError_SaveList;
 122   if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
 123     return CSR_AArch64_AAPCS_SwiftTail_SaveList;
 124   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
 125     return CSR_AArch64_RT_MostRegs_SaveList;
 126   if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
 127     return CSR_AArch64_RT_AllRegs_SaveList;
 128   if (MF->getFunction().getCallingConv() == CallingConv::Win64)
 129     // This is for OSes other than Windows; Windows is a separate case further
 130     // above.
 131     return CSR_AArch64_AAPCS_X18_SaveList;
 132   if (MF->getInfo<AArch64FunctionInfo>()->isSVECC())
 133     return CSR_AArch64_SVE_AAPCS_SaveList;
 134   return CSR_AArch64_AAPCS_SaveList;
 135 }
 136
 137 const MCPhysReg *
 138 AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const {
 139   assert(MF && "Invalid MachineFunction pointer.");
 140   assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
 141          "Invalid subtarget for getDarwinCalleeSavedRegs");
 142
 143   if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check)
 144     report_fatal_error(
 145         "Calling convention CFGuard_Check is unsupported on Darwin.");
 146   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall)
 147     return CSR_Darwin_AArch64_AAVPCS_SaveList;
 148   if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall)
 149     report_fatal_error(
 150         "Calling convention SVE_VectorCall is unsupported on Darwin.");
 151   if (MF->getFunction().getCallingConv() ==
 152           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
 153     report_fatal_error(
 154         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
 155         "only supported to improve calls to SME ACLE save/restore/disable-za "
 156         "functions, and is not intended to be used beyond that scope.");
 157   if (MF->getFunction().getCallingConv() ==
 158           CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
 159     report_fatal_error(
 160         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
 161         "only supported to improve calls to SME ACLE __arm_sme_state "
 162         "and is not intended to be used beyond that scope.");
 163   if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS)
 164     return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()
 165                ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList
 166                : CSR_Darwin_AArch64_CXX_TLS_SaveList;
 167   if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering()
 168           ->supportSwiftError() &&
 169       MF->getFunction().getAttributes().hasAttrSomewhere(
 170           Attribute::SwiftError))
 171     return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;
 172   if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail)
 173     return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;
 174   if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
 175     return CSR_Darwin_AArch64_RT_MostRegs_SaveList;
 176   if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll)
 177     return CSR_Darwin_AArch64_RT_AllRegs_SaveList;
 178   if (MF->getFunction().getCallingConv() == CallingConv::Win64)
 179     return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;
 180   return CSR_Darwin_AArch64_AAPCS_SaveList;
 181 }
 182
 183 const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(
 184     const MachineFunction *MF) const {
 185   assert(MF && "Invalid MachineFunction pointer.");
 186   if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
 187       MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())
 188     return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;
 189   return nullptr;
 190 }
 191
 192 void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(
 193     MachineFunction &MF) const {
 194   const MCPhysReg *CSRs = getCalleeSavedRegs(&MF);
 195   SmallVector<MCPhysReg, 32> UpdatedCSRs;
 196   for (const MCPhysReg *I = CSRs; *I; ++I)
 197     UpdatedCSRs.push_back(*I);
 198
 199   for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
 200     if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
 201       UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i));
 202     }
 203   }
 204   // Register lists are zero-terminated.
 205   UpdatedCSRs.push_back(0);
 206   MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);
 207 }
 208
 209 const TargetRegisterClass *
 210 AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,
 211                                        unsigned Idx) const {
 212   // edge case for GPR/FPR register classes
 213   if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)
 214     return &AArch64::FPR32RegClass;
 215   else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)
 216     return &AArch64::FPR64RegClass;
 217
 218   // Forward to TableGen's default version.
 219   return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);
 220 }
 221
 222 const uint32_t *
 223 AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,
 224                                                 CallingConv::ID CC) const {
 225   assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&
 226          "Invalid subtarget for getDarwinCallPreservedMask");
 227
 228   if (CC == CallingConv::CXX_FAST_TLS)
 229     return CSR_Darwin_AArch64_CXX_TLS_RegMask;
 230   if (CC == CallingConv::AArch64_VectorCall)
 231     return CSR_Darwin_AArch64_AAVPCS_RegMask;
 232   if (CC == CallingConv::AArch64_SVE_VectorCall)
 233     report_fatal_error(
 234         "Calling convention SVE_VectorCall is unsupported on Darwin.");
 235   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
 236     report_fatal_error(
 237         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is "
 238         "unsupported on Darwin.");
 239   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
 240     report_fatal_error(
 241         "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "
 242         "unsupported on Darwin.");
 243   if (CC == CallingConv::CFGuard_Check)
 244     report_fatal_error(
 245         "Calling convention CFGuard_Check is unsupported on Darwin.");
 246   if (MF.getSubtarget<AArch64Subtarget>()
 247           .getTargetLowering()
 248           ->supportSwiftError() &&
 249       MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
 250     return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;
 251   if (CC == CallingConv::SwiftTail)
 252     return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;
 253   if (CC == CallingConv::PreserveMost)
 254     return CSR_Darwin_AArch64_RT_MostRegs_RegMask;
 255   if (CC == CallingConv::PreserveAll)
 256     return CSR_Darwin_AArch64_RT_AllRegs_RegMask;
 257   return CSR_Darwin_AArch64_AAPCS_RegMask;
 258 }
 259
 260 const uint32_t *
 261 AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,
 262                                           CallingConv::ID CC) const {
 263   bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);
 264   if (CC == CallingConv::GHC)
 265     // This is academic because all GHC calls are (supposed to be) tail calls
 266     return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;
 267   if (CC == CallingConv::AnyReg)
 268     return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;
 269
 270   // All the following calling conventions are handled differently on Darwin.
 271   if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
 272     if (SCS)
 273       report_fatal_error("ShadowCallStack attribute not supported on Darwin.");
 274     return getDarwinCallPreservedMask(MF, CC);
 275   }
 276
 277   if (CC == CallingConv::AArch64_VectorCall)
 278     return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;
 279   if (CC == CallingConv::AArch64_SVE_VectorCall)
 280     return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask
 281                : CSR_AArch64_SVE_AAPCS_RegMask;
 282   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)
 283     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
 284   if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)
 285     return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;
 286   if (CC == CallingConv::CFGuard_Check)
 287     return CSR_Win_AArch64_CFGuard_Check_RegMask;
 288   if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()
 289           ->supportSwiftError() &&
 290       MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))
 291     return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask
 292                : CSR_AArch64_AAPCS_SwiftError_RegMask;
 293   if (CC == CallingConv::SwiftTail) {
 294     if (SCS)
 295       report_fatal_error("ShadowCallStack attribute not supported with swifttail");
 296     return CSR_AArch64_AAPCS_SwiftTail_RegMask;
 297   }
 298   if (CC == CallingConv::PreserveMost)
 299     return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask
 300                : CSR_AArch64_RT_MostRegs_RegMask;
 301   else if (CC == CallingConv::PreserveAll)
 302     return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask
 303                : CSR_AArch64_RT_AllRegs_RegMask;
 304
 305   else
 306     return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;
 307 }
 308
 309 const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(
 310     const MachineFunction &MF) const {
 311   if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())
 312     return CSR_AArch64_AAPCS_RegMask;
 313
 314   return nullptr;
 315 }
 316
 317 const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {
 318   if (TT.isOSDarwin())
 319     return CSR_Darwin_AArch64_TLS_RegMask;
 320
 321   assert(TT.isOSBinFormatELF() && "Invalid target");
 322   return CSR_AArch64_TLS_ELF_RegMask;
 323 }
 324
 325 void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,
 326                                                  const uint32_t **Mask) const {
 327   uint32_t *UpdatedMask = MF.allocateRegMask();
 328   unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs());
 329   memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize);
 330
 331   for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {
 332     if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {
 333       for (MCPhysReg SubReg :
 334            subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) {
 335         // See TargetRegisterInfo::getCallPreservedMask for how to interpret the
 336         // register mask.
 337         UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32);
 338       }
 339     }
 340   }
 341   *Mask = UpdatedMask;
 342 }
 343
 344 const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const {
 345   return CSR_AArch64_SMStartStop_RegMask;
 346 }
 347
 348 const uint32_t *
 349 AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const {
 350   return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;
 351 }
 352
 353 const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {
 354   return CSR_AArch64_NoRegs_RegMask;
 355 }
 356
 357 const uint32_t *
 358 AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,
 359                                                 CallingConv::ID CC) const {
 360   // This should return a register mask that is the same as that returned by
 361   // getCallPreservedMask but that additionally preserves the register used for
 362   // the first i64 argument (which must also be the register used to return a
 363   // single i64 return value)
 364   //
 365   // In case that the calling convention does not use the same register for
 366   // both, the function should return NULL (does not currently apply)
 367   assert(CC != CallingConv::GHC && "should not be GHC calling convention.");
 368   if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())
 369     return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;
 370   return CSR_AArch64_AAPCS_ThisReturn_RegMask;
 371 }
 372
 373 const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {
 374   return CSR_AArch64_StackProbe_Windows_RegMask;
 375 }
 376
 377 std::optional<std::string>
 378 AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,
 379                                         MCRegister PhysReg) const {
 380   if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19))
 381     return std::string("X19 is used as the frame base pointer register.");
 382
 383   if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
 384     bool warn = false;
 385     if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) ||
 386         MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) ||
 387         MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) ||
 388         MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) ||
 389         MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28))
 390       warn = true;
 391
 392     for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
 393       if (MCRegisterInfo::regsOverlap(PhysReg, i))
 394         warn = true;
 395
 396     if (warn)
 397       return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) +
 398              " is clobbered by asynchronous signals when using Arm64EC.";
 399   }
 400
 401   return {};
 402 }
 403
 404 BitVector
 405 AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
 406   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 407
 408   // FIXME: avoid re-calculating this every time.
 409   BitVector Reserved(getNumRegs());
 410   markSuperRegs(Reserved, AArch64::WSP);
 411   markSuperRegs(Reserved, AArch64::WZR);
 412
 413   if (TFI->hasFP(MF) || TT.isOSDarwin())
 414     markSuperRegs(Reserved, AArch64::W29);
 415
 416   if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {
 417     // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous
 418     // signals, so we can't ever use them.
 419     markSuperRegs(Reserved, AArch64::W13);
 420     markSuperRegs(Reserved, AArch64::W14);
 421     markSuperRegs(Reserved, AArch64::W23);
 422     markSuperRegs(Reserved, AArch64::W24);
 423     markSuperRegs(Reserved, AArch64::W28);
 424     for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)
 425       markSuperRegs(Reserved, i);
 426   }
 427
 428   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
 429     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))
 430       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
 431   }
 432
 433   if (hasBasePointer(MF))
 434     markSuperRegs(Reserved, AArch64::W19);
 435
 436   // SLH uses register W16/X16 as the taint register.
 437   if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
 438     markSuperRegs(Reserved, AArch64::W16);
 439
 440   // FFR is modelled as global state that cannot be allocated.
 441   if (MF.getSubtarget<AArch64Subtarget>().hasSVE())
 442     Reserved.set(AArch64::FFR);
 443
 444   // SME tiles are not allocatable.
 445   if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {
 446     for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA))
 447       Reserved.set(SubReg);
 448   }
 449
 450   // VG cannot be allocated
 451   Reserved.set(AArch64::VG);
 452
 453   if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) {
 454     for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true);
 455          SubReg.isValid(); ++SubReg)
 456       Reserved.set(*SubReg);
 457   }
 458
 459   markSuperRegs(Reserved, AArch64::FPCR);
 460   markSuperRegs(Reserved, AArch64::FPSR);
 461
 462   if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
 463     markSuperRegs(Reserved, AArch64::X27);
 464     markSuperRegs(Reserved, AArch64::X28);
 465     markSuperRegs(Reserved, AArch64::W27);
 466     markSuperRegs(Reserved, AArch64::W28);
 467   }
 468
 469   assert(checkAllSuperRegsMarked(Reserved));
 470   return Reserved;
 471 }
 472
 473 BitVector
 474 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 475   BitVector Reserved = getStrictlyReservedRegs(MF);
 476
 477   for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {
 478     if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))
 479       markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));
 480   }
 481
 482   assert(checkAllSuperRegsMarked(Reserved));
 483   return Reserved;
 484 }
 485
 486 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
 487                                         MCRegister Reg) const {
 488   return getReservedRegs(MF)[Reg];
 489 }
 490
 491 bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF,
 492                                                 MCRegister Reg) const {
 493   return getStrictlyReservedRegs(MF)[Reg];
 494 }
 495
 496 bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
 497   return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) {
 498     return isStrictlyReservedReg(MF, r);
 499   });
 500 }
 501
 502 void AArch64RegisterInfo::emitReservedArgRegCallError(
 503     const MachineFunction &MF) const {
 504   const Function &F = MF.getFunction();
 505   F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"
 506     " function calls if any of the argument registers is reserved.")});
 507 }
 508
 509 bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,
 510                                           MCRegister PhysReg) const {
 511   // SLH uses register X16 as the taint register but it will fallback to a different
 512   // method if the user clobbers it. So X16 is not reserved for inline asm but is
 513   // for normal codegen.
 514   if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) &&
 515         MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16))
 516     return true;
 517
 518   // ZA/ZT0 registers are reserved but may be permitted in the clobber list.
 519   if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0)
 520     return true;
 521
 522   return !isReservedReg(MF, PhysReg);
 523 }
 524
 525 const TargetRegisterClass *
 526 AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF,
 527                                       unsigned Kind) const {
 528   return &AArch64::GPR64spRegClass;
 529 }
 530
 531 const TargetRegisterClass *
 532 AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {
 533   if (RC == &AArch64::CCRRegClass)
 534     return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.
 535   return RC;
 536 }
 537
 538 unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }
 539
 540 bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {
 541   const MachineFrameInfo &MFI = MF.getFrameInfo();
 542
 543   // In the presence of variable sized objects or funclets, if the fixed stack
 544   // size is large enough that referencing from the FP won't result in things
 545   // being in range relatively often, we can use a base pointer to allow access
 546   // from the other direction like the SP normally works.
 547   //
 548   // Furthermore, if both variable sized objects are present, and the
 549   // stack needs to be dynamically re-aligned, the base pointer is the only
 550   // reliable way to reference the locals.
 551   if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {
 552     if (hasStackRealignment(MF))
 553       return true;
 554
 555     if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
 556       const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 557       // Frames that have variable sized objects and scalable SVE objects,
 558       // should always use a basepointer.
 559       if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE())
 560         return true;
 561     }
 562
 563     // Conservatively estimate whether the negative offset from the frame
 564     // pointer will be sufficient to reach. If a function has a smallish
 565     // frame, it's less likely to have lots of spills and callee saved
 566     // space, so it's all more likely to be within range of the frame pointer.
 567     // If it's wrong, we'll materialize the constant and still get to the
 568     // object; it's just suboptimal. Negative offsets use the unscaled
 569     // load/store instructions, which have a 9-bit signed immediate.
 570     return MFI.getLocalFrameSize() >= 256;
 571   }
 572
 573   return false;
 574 }
 575
 576 bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,
 577                                              MCRegister Reg) const {
 578   CallingConv::ID CC = MF.getFunction().getCallingConv();
 579   const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
 580   bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv());
 581
 582   auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {
 583     return llvm::is_contained(RegList, Reg);
 584   };
 585
 586   switch (CC) {
 587   default:
 588     report_fatal_error("Unsupported calling convention.");
 589   case CallingConv::GHC:
 590     return HasReg(CC_AArch64_GHC_ArgRegs, Reg);
 591   case CallingConv::C:
 592   case CallingConv::Fast:
 593   case CallingConv::PreserveMost:
 594   case CallingConv::PreserveAll:
 595   case CallingConv::CXX_FAST_TLS:
 596   case CallingConv::Swift:
 597   case CallingConv::SwiftTail:
 598   case CallingConv::Tail:
 599     if (STI.isTargetWindows()) {
 600       if (IsVarArg)
 601         return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
 602       switch (CC) {
 603       default:
 604         return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
 605       case CallingConv::Swift:
 606       case CallingConv::SwiftTail:
 607         return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) ||
 608                HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
 609       }
 610     }
 611     if (!STI.isTargetDarwin()) {
 612       switch (CC) {
 613       default:
 614         return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
 615       case CallingConv::Swift:
 616       case CallingConv::SwiftTail:
 617         return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||
 618                HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);
 619       }
 620     }
 621     if (!IsVarArg) {
 622       switch (CC) {
 623       default:
 624         return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);
 625       case CallingConv::Swift:
 626       case CallingConv::SwiftTail:
 627         return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||
 628                HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);
 629       }
 630     }
 631     if (STI.isTargetILP32())
 632       return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);
 633     return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);
 634   case CallingConv::Win64:
 635     if (IsVarArg)
 636       HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);
 637     return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
 638   case CallingConv::CFGuard_Check:
 639     return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);
 640   case CallingConv::AArch64_VectorCall:
 641   case CallingConv::AArch64_SVE_VectorCall:
 642   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:
 643   case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:
 644     if (STI.isTargetWindows())
 645       return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);
 646     return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);
 647   }
 648 }
 649
 650 Register
 651 AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
 652   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 653   return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;
 654 }
 655
 656 bool AArch64RegisterInfo::requiresRegisterScavenging(
 657     const MachineFunction &MF) const {
 658   return true;
 659 }
 660
 661 bool AArch64RegisterInfo::requiresVirtualBaseRegisters(
 662     const MachineFunction &MF) const {
 663   return true;
 664 }
 665
 666 bool
 667 AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {
 668   // This function indicates whether the emergency spillslot should be placed
 669   // close to the beginning of the stackframe (closer to FP) or the end
 670   // (closer to SP).
 671   //
 672   // The beginning works most reliably if we have a frame pointer.
 673   // In the presence of any non-constant space between FP and locals,
 674   // (e.g. in case of stack realignment or a scalable SVE area), it is
 675   // better to use SP or BP.
 676   const AArch64FrameLowering &TFI = *getFrameLowering(MF);
 677   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 678   assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||
 679           AFI->hasCalculatedStackSizeSVE()) &&
 680          "Expected SVE area to be calculated by this point");
 681   return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE();
 682 }
 683
 684 bool AArch64RegisterInfo::requiresFrameIndexScavenging(
 685     const MachineFunction &MF) const {
 686   return true;
 687 }
 688
 689 bool
 690 AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {
 691   const MachineFrameInfo &MFI = MF.getFrameInfo();
 692   if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack())
 693     return true;
 694   return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();
 695 }
 696
 697 /// needsFrameBaseReg - Returns true if the instruction's frame index
 698 /// reference would be better served by a base register other than FP
 699 /// or SP. Used by LocalStackFrameAllocation to determine which frame index
 700 /// references it should create new base registers for.
 701 bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,
 702                                             int64_t Offset) const {
 703   for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)
 704     assert(i < MI->getNumOperands() &&
 705            "Instr doesn't have FrameIndex operand!");
 706
 707   // It's the load/store FI references that cause issues, as it can be difficult
 708   // to materialize the offset if it won't fit in the literal field. Estimate
 709   // based on the size of the local frame and some conservative assumptions
 710   // about the rest of the stack frame (note, this is pre-regalloc, so
 711   // we don't know everything for certain yet) whether this offset is likely
 712   // to be out of range of the immediate. Return true if so.
 713
 714   // We only generate virtual base registers for loads and stores, so
 715   // return false for everything else.
 716   if (!MI->mayLoad() && !MI->mayStore())
 717     return false;
 718
 719   // Without a virtual base register, if the function has variable sized
 720   // objects, all fixed-size local references will be via the frame pointer,
 721   // Approximate the offset and see if it's legal for the instruction.
 722   // Note that the incoming offset is based on the SP value at function entry,
 723   // so it'll be negative.
 724   MachineFunction &MF = *MI->getParent()->getParent();
 725   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 726   MachineFrameInfo &MFI = MF.getFrameInfo();
 727
 728   // Estimate an offset from the frame pointer.
 729   // Conservatively assume all GPR callee-saved registers get pushed.
 730   // FP, LR, X19-X28, D8-D15. 64-bits each.
 731   int64_t FPOffset = Offset - 16 * 20;
 732   // Estimate an offset from the stack pointer.
 733   // The incoming offset is relating to the SP at the start of the function,
 734   // but when we access the local it'll be relative to the SP after local
 735   // allocation, so adjust our SP-relative offset by that allocation size.
 736   Offset += MFI.getLocalFrameSize();
 737   // Assume that we'll have at least some spill slots allocated.
 738   // FIXME: This is a total SWAG number. We should run some statistics
 739   //        and pick a real one.
 740   Offset += 128; // 128 bytes of spill slots
 741
 742   // If there is a frame pointer, try using it.
 743   // The FP is only available if there is no dynamic realignment. We
 744   // don't know for sure yet whether we'll need that, so we guess based
 745   // on whether there are any local variables that would trigger it.
 746   if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset))
 747     return false;
 748
 749   // If we can reference via the stack pointer or base pointer, try that.
 750   // FIXME: This (and the code that resolves the references) can be improved
 751   //        to only disallow SP relative references in the live range of
 752   //        the VLA(s). In practice, it's unclear how much difference that
 753   //        would make, but it may be worth doing.
 754   if (isFrameOffsetLegal(MI, AArch64::SP, Offset))
 755     return false;
 756
 757   // If even offset 0 is illegal, we don't want a virtual base register.
 758   if (!isFrameOffsetLegal(MI, AArch64::SP, 0))
 759     return false;
 760
 761   // The offset likely isn't legal; we want to allocate a virtual base register.
 762   return true;
 763 }
 764
 765 bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,
 766                                              Register BaseReg,
 767                                              int64_t Offset) const {
 768   assert(MI && "Unable to get the legal offset for nil instruction.");
 769   StackOffset SaveOffset = StackOffset::getFixed(Offset);
 770   return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal;
 771 }
 772
 773 /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx
 774 /// at the beginning of the basic block.
 775 Register
 776 AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
 777                                                   int FrameIdx,
 778                                                   int64_t Offset) const {
 779   MachineBasicBlock::iterator Ins = MBB->begin();
 780   DebugLoc DL; // Defaults to "unknown"
 781   if (Ins != MBB->end())
 782     DL = Ins->getDebugLoc();
 783   const MachineFunction &MF = *MBB->getParent();
 784   const AArch64InstrInfo *TII =
 785       MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
 786   const MCInstrDesc &MCID = TII->get(AArch64::ADDXri);
 787   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
 788   Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
 789   MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0, this, MF));
 790   unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
 791
 792   BuildMI(*MBB, Ins, DL, MCID, BaseReg)
 793       .addFrameIndex(FrameIdx)
 794       .addImm(Offset)
 795       .addImm(Shifter);
 796
 797   return BaseReg;
 798 }
 799
 800 void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,
 801                                             int64_t Offset) const {
 802   // ARM doesn't need the general 64-bit offsets
 803   StackOffset Off = StackOffset::getFixed(Offset);
 804
 805   unsigned i = 0;
 806   while (!MI.getOperand(i).isFI()) {
 807     ++i;
 808     assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");
 809   }
 810
 811   const MachineFunction *MF = MI.getParent()->getParent();
 812   const AArch64InstrInfo *TII =
 813       MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
 814   bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII);
 815   assert(Done && "Unable to resolve frame index!");
 816   (void)Done;
 817 }
 818
 819 // Create a scratch register for the frame index elimination in an instruction.
 820 // This function has special handling of stack tagging loop pseudos, in which
 821 // case it can also change the instruction opcode.
 822 static Register
 823 createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,
 824                                     const AArch64InstrInfo *TII) {
 825   // ST*Gloop have a reserved scratch register in operand 1. Use it, and also
 826   // replace the instruction with the writeback variant because it will now
 827   // satisfy the operand constraints for it.
 828   Register ScratchReg;
 829   if (MI.getOpcode() == AArch64::STGloop ||
 830       MI.getOpcode() == AArch64::STZGloop) {
 831     assert(FIOperandNum == 3 &&
 832            "Wrong frame index operand for STGloop/STZGloop");
 833     unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback
 834                                                      : AArch64::STZGloop_wback;
 835     ScratchReg = MI.getOperand(1).getReg();
 836     MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true);
 837     MI.setDesc(TII->get(Op));
 838     MI.tieOperands(1, 3);
 839   } else {
 840     ScratchReg =
 841         MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
 842     MI.getOperand(FIOperandNum)
 843         .ChangeToRegister(ScratchReg, false, false, true);
 844   }
 845   return ScratchReg;
 846 }
 847
 848 void AArch64RegisterInfo::getOffsetOpcodes(
 849     const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {
 850   // The smallest scalable element supported by scaled SVE addressing
 851   // modes are predicates, which are 2 scalable bytes in size. So the scalable
 852   // byte offset must always be a multiple of 2.
 853   assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
 854
 855   // Add fixed-sized offset using existing DIExpression interface.
 856   DIExpression::appendOffset(Ops, Offset.getFixed());
 857
 858   unsigned VG = getDwarfRegNum(AArch64::VG, true);
 859   int64_t VGSized = Offset.getScalable() / 2;
 860   if (VGSized > 0) {
 861     Ops.push_back(dwarf::DW_OP_constu);
 862     Ops.push_back(VGSized);
 863     Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
 864     Ops.push_back(dwarf::DW_OP_mul);
 865     Ops.push_back(dwarf::DW_OP_plus);
 866   } else if (VGSized < 0) {
 867     Ops.push_back(dwarf::DW_OP_constu);
 868     Ops.push_back(-VGSized);
 869     Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});
 870     Ops.push_back(dwarf::DW_OP_mul);
 871     Ops.push_back(dwarf::DW_OP_minus);
 872   }
 873 }
 874
 875 bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 876                                               int SPAdj, unsigned FIOperandNum,
 877                                               RegScavenger *RS) const {
 878   assert(SPAdj == 0 && "Unexpected");
 879
 880   MachineInstr &MI = *II;
 881   MachineBasicBlock &MBB = *MI.getParent();
 882   MachineFunction &MF = *MBB.getParent();
 883   const MachineFrameInfo &MFI = MF.getFrameInfo();
 884   const AArch64InstrInfo *TII =
 885       MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
 886   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 887   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 888   bool Tagged =
 889       MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;
 890   Register FrameReg;
 891
 892   // Special handling of dbg_value, stackmap patchpoint statepoint instructions.
 893   if (MI.getOpcode() == TargetOpcode::STACKMAP ||
 894       MI.getOpcode() == TargetOpcode::PATCHPOINT ||
 895       MI.getOpcode() == TargetOpcode::STATEPOINT) {
 896     StackOffset Offset =
 897         TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg,
 898                                         /*PreferFP=*/true,
 899                                         /*ForSimm=*/false);
 900     Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());
 901     MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/);
 902     MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed());
 903     return false;
 904   }
 905
 906   if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
 907     MachineOperand &FI = MI.getOperand(FIOperandNum);
 908     StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex);
 909     assert(!Offset.getScalable() &&
 910            "Frame offsets with a scalable component are not supported");
 911     FI.ChangeToImmediate(Offset.getFixed());
 912     return false;
 913   }
 914
 915   StackOffset Offset;
 916   if (MI.getOpcode() == AArch64::TAGPstack) {
 917     // TAGPstack must use the virtual frame register in its 3rd operand.
 918     const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
 919     FrameReg = MI.getOperand(3).getReg();
 920     Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
 921                                       AFI->getTaggedBasePointerOffset());
 922   } else if (Tagged) {
 923     StackOffset SPOffset = StackOffset::getFixed(
 924         MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize());
 925     if (MFI.hasVarSizedObjects() ||
 926         isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) !=
 927             (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {
 928       // Can't update to SP + offset in place. Precalculate the tagged pointer
 929       // in a scratch register.
 930       Offset = TFI->resolveFrameIndexReference(
 931           MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
 932       Register ScratchReg =
 933           MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
 934       emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset,
 935                       TII);
 936       BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg)
 937           .addReg(ScratchReg)
 938           .addReg(ScratchReg)
 939           .addImm(0);
 940       MI.getOperand(FIOperandNum)
 941           .ChangeToRegister(ScratchReg, false, false, true);
 942       return false;
 943     }
 944     FrameReg = AArch64::SP;
 945     Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +
 946                                    (int64_t)MFI.getStackSize());
 947   } else {
 948     Offset = TFI->resolveFrameIndexReference(
 949         MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);
 950   }
 951
 952   // Modify MI as necessary to handle as much of 'Offset' as possible
 953   if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII))
 954     return true;
 955
 956   assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&
 957          "Emergency spill slot is out of reach");
 958
 959   // If we get here, the immediate doesn't fit into the instruction.  We folded
 960   // as much as possible above.  Handle the rest, providing a register that is
 961   // SP+LargeImm.
 962   Register ScratchReg =
 963       createScratchRegisterForInstruction(MI, FIOperandNum, TII);
 964   emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII);
 965   return false;
 966 }
 967
 968 unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
 969                                                   MachineFunction &MF) const {
 970   const AArch64FrameLowering *TFI = getFrameLowering(MF);
 971
 972   switch (RC->getID()) {
 973   default:
 974     return 0;
 975   case AArch64::GPR32RegClassID:
 976   case AArch64::GPR32spRegClassID:
 977   case AArch64::GPR32allRegClassID:
 978   case AArch64::GPR64spRegClassID:
 979   case AArch64::GPR64allRegClassID:
 980   case AArch64::GPR64RegClassID:
 981   case AArch64::GPR32commonRegClassID:
 982   case AArch64::GPR64commonRegClassID:
 983     return 32 - 1                                   // XZR/SP
 984               - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP
 985               - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()
 986               - hasBasePointer(MF);  // X19
 987   case AArch64::FPR8RegClassID:
 988   case AArch64::FPR16RegClassID:
 989   case AArch64::FPR32RegClassID:
 990   case AArch64::FPR64RegClassID:
 991   case AArch64::FPR128RegClassID:
 992     return 32;
 993
 994   case AArch64::MatrixIndexGPR32_8_11RegClassID:
 995   case AArch64::MatrixIndexGPR32_12_15RegClassID:
 996     return 4;
 997
 998   case AArch64::DDRegClassID:
 999   case AArch64::DDDRegClassID:
1000   case AArch64::DDDDRegClassID:
1001   case AArch64::QQRegClassID:
1002   case AArch64::QQQRegClassID:
1003   case AArch64::QQQQRegClassID:
1004     return 32;
1005
1006   case AArch64::FPR128_loRegClassID:
1007   case AArch64::FPR64_loRegClassID:
1008   case AArch64::FPR16_loRegClassID:
1009     return 16;
1010   case AArch64::FPR128_0to7RegClassID:
1011     return 8;
1012   }
1013 }
1014
1015 unsigned AArch64RegisterInfo::getLocalAddressRegister(
1016   const MachineFunction &MF) const {
1017   const auto &MFI = MF.getFrameInfo();
1018   if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
1019     return AArch64::SP;
1020   else if (hasStackRealignment(MF))
1021     return getBaseRegister();
1022   return getFrameRegister(MF);
1023 }
1024
1025 /// SrcRC and DstRC will be morphed into NewRC if this returns true
1026 bool AArch64RegisterInfo::shouldCoalesce(
1027     MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
1028     const TargetRegisterClass *DstRC, unsigned DstSubReg,
1029     const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
1030   MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
1031
1032   if (MI->isCopy() &&
1033       ((DstRC->getID() == AArch64::GPR64RegClassID) ||
1034        (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
1035       MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg())
1036     // Do not coalesce in the case of a 32-bit subregister copy
1037     // which implements a 32 to 64 bit zero extension
1038     // which relies on the upper 32 bits being zeroed.
1039     return false;
1040
1041   auto IsCoalescerBarrier = [](const MachineInstr &MI) {
1042     switch (MI.getOpcode()) {
1043     case AArch64::COALESCER_BARRIER_FPR16:
1044     case AArch64::COALESCER_BARRIER_FPR32:
1045     case AArch64::COALESCER_BARRIER_FPR64:
1046     case AArch64::COALESCER_BARRIER_FPR128:
1047       return true;
1048     default:
1049       return false;
1050     }
1051   };
1052
1053   // For calls that temporarily have to toggle streaming mode as part of the
1054   // call-sequence, we need to be more careful when coalescing copy instructions
1055   // so that we don't end up coalescing the NEON/FP result or argument register
1056   // with a whole Z-register, such that after coalescing the register allocator
1057   // will try to spill/reload the entire Z register.
1058   //
1059   // We do this by checking if the node has any defs/uses that are
1060   // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to
1061   // instruct the coalescer to avoid coalescing the copy.
1062   if (MI->isCopy() && SubReg != DstSubReg &&
1063       (AArch64::ZPRRegClass.hasSubClassEq(DstRC) ||
1064        AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) {
1065     unsigned SrcReg = MI->getOperand(1).getReg();
1066     if (any_of(MRI.def_instructions(SrcReg), IsCoalescerBarrier))
1067       return false;
1068     unsigned DstReg = MI->getOperand(0).getReg();
1069     if (any_of(MRI.use_nodbg_instructions(DstReg), IsCoalescerBarrier))
1070       return false;
1071   }
1072
1073   return true;
1074 }
1075
1076 bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo(
1077     MCRegister R) const {
1078   return R == AArch64::VG;
1079 }