lib/Target/AArch64/AArch64RegisterBankInfo.cpp

   1 //===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 /// This file implements the targeting of the RegisterBankInfo class for
  10 /// AArch64.
  11 /// \todo This should be generated by TableGen.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64RegisterBankInfo.h"
  15 #include "AArch64InstrInfo.h"
  16 #include "llvm/ADT/SmallVector.h"
  17 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
  18 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  19 #include "llvm/CodeGen/LowLevelType.h"
  20 #include "llvm/CodeGen/MachineFunction.h"
  21 #include "llvm/CodeGen/MachineInstr.h"
  22 #include "llvm/CodeGen/MachineOperand.h"
  23 #include "llvm/CodeGen/MachineRegisterInfo.h"
  24 #include "llvm/CodeGen/TargetOpcodes.h"
  25 #include "llvm/CodeGen/TargetRegisterInfo.h"
  26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  27 #include "llvm/Support/ErrorHandling.h"
  28 #include <algorithm>
  29 #include <cassert>
  30
  31 #define GET_TARGET_REGBANK_IMPL
  32 #include "AArch64GenRegisterBank.inc"
  33
  34 // This file will be TableGen'ed at some point.
  35 #include "AArch64GenRegisterBankInfo.def"
  36
  37 using namespace llvm;
  38
  39 AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI)
  40     : AArch64GenRegisterBankInfo() {
  41   static bool AlreadyInit = false;
  42   // We have only one set of register banks, whatever the subtarget
  43   // is. Therefore, the initialization of the RegBanks table should be
  44   // done only once. Indeed the table of all register banks
  45   // (AArch64::RegBanks) is unique in the compiler. At some point, it
  46   // will get tablegen'ed and the whole constructor becomes empty.
  47   if (AlreadyInit)
  48     return;
  49   AlreadyInit = true;
  50
  51   const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID);
  52   (void)RBGPR;
  53   assert(&AArch64::GPRRegBank == &RBGPR &&
  54          "The order in RegBanks is messed up");
  55
  56   const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID);
  57   (void)RBFPR;
  58   assert(&AArch64::FPRRegBank == &RBFPR &&
  59          "The order in RegBanks is messed up");
  60
  61   const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID);
  62   (void)RBCCR;
  63   assert(&AArch64::CCRegBank == &RBCCR && "The order in RegBanks is messed up");
  64
  65   // The GPR register bank is fully defined by all the registers in
  66   // GR64all + its subclasses.
  67   assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&
  68          "Subclass not added?");
  69   assert(RBGPR.getSize() == 64 && "GPRs should hold up to 64-bit");
  70
  71   // The FPR register bank is fully defined by all the registers in
  72   // GR64all + its subclasses.
  73   assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&
  74          "Subclass not added?");
  75   assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&
  76          "Subclass not added?");
  77   assert(RBFPR.getSize() == 512 &&
  78          "FPRs should hold up to 512-bit via QQQQ sequence");
  79
  80   assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&
  81          "Class not added?");
  82   assert(RBCCR.getSize() == 32 && "CCR should hold up to 32-bit");
  83
  84   // Check that the TableGen'ed like file is in sync we our expectations.
  85   // First, the Idx.
  86   assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,
  87                                 {PMI_GPR32, PMI_GPR64}) &&
  88          "PartialMappingIdx's are incorrectly ordered");
  89   assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,
  90                                 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,
  91                                  PMI_FPR256, PMI_FPR512}) &&
  92          "PartialMappingIdx's are incorrectly ordered");
  93 // Now, the content.
  94 // Check partial mapping.
  95 #define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB)                      \
  96   do {                                                                         \
  97     assert(                                                                    \
  98         checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \
  99         #Idx " is incorrectly initialized");                                   \
 100   } while (false)
 101
 102   CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);
 103   CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);
 104   CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);
 105   CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);
 106   CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);
 107   CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);
 108   CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);
 109   CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);
 110
 111 // Check value mapping.
 112 #define CHECK_VALUEMAP_IMPL(RBName, Size, Offset)                              \
 113   do {                                                                         \
 114     assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size,            \
 115                              PartialMappingIdx::PMI_First##RBName, Size,       \
 116                              Offset) &&                                        \
 117            #RBName #Size " " #Offset " is incorrectly initialized");           \
 118   } while (false)
 119
 120 #define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)
 121
 122   CHECK_VALUEMAP(GPR, 32);
 123   CHECK_VALUEMAP(GPR, 64);
 124   CHECK_VALUEMAP(FPR, 16);
 125   CHECK_VALUEMAP(FPR, 32);
 126   CHECK_VALUEMAP(FPR, 64);
 127   CHECK_VALUEMAP(FPR, 128);
 128   CHECK_VALUEMAP(FPR, 256);
 129   CHECK_VALUEMAP(FPR, 512);
 130
 131 // Check the value mapping for 3-operands instructions where all the operands
 132 // map to the same value mapping.
 133 #define CHECK_VALUEMAP_3OPS(RBName, Size)                                      \
 134   do {                                                                         \
 135     CHECK_VALUEMAP_IMPL(RBName, Size, 0);                                      \
 136     CHECK_VALUEMAP_IMPL(RBName, Size, 1);                                      \
 137     CHECK_VALUEMAP_IMPL(RBName, Size, 2);                                      \
 138   } while (false)
 139
 140   CHECK_VALUEMAP_3OPS(GPR, 32);
 141   CHECK_VALUEMAP_3OPS(GPR, 64);
 142   CHECK_VALUEMAP_3OPS(FPR, 32);
 143   CHECK_VALUEMAP_3OPS(FPR, 64);
 144   CHECK_VALUEMAP_3OPS(FPR, 128);
 145   CHECK_VALUEMAP_3OPS(FPR, 256);
 146   CHECK_VALUEMAP_3OPS(FPR, 512);
 147
 148 #define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size)                 \
 149   do {                                                                         \
 150     unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min;               \
 151     unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min;               \
 152     (void)PartialMapDstIdx;                                                    \
 153     (void)PartialMapSrcIdx;                                                    \
 154     const ValueMapping *Map = getCopyMapping(                                  \
 155         AArch64::RBNameDst##RegBankID, AArch64::RBNameSrc##RegBankID, Size);  \
 156     (void)Map;                                                                 \
 157     assert(Map[0].BreakDown ==                                                 \
 158                &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] &&  \
 159            Map[0].NumBreakDowns == 1 && #RBNameDst #Size                       \
 160            " Dst is incorrectly initialized");                                 \
 161     assert(Map[1].BreakDown ==                                                 \
 162                &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] &&  \
 163            Map[1].NumBreakDowns == 1 && #RBNameSrc #Size                       \
 164            " Src is incorrectly initialized");                                 \
 165                                                                                \
 166   } while (false)
 167
 168   CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);
 169   CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);
 170   CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);
 171   CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);
 172   CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);
 173   CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);
 174   CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);
 175   CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);
 176
 177 #define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize)                                 \
 178   do {                                                                         \
 179     unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min;                    \
 180     unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min;                    \
 181     (void)PartialMapDstIdx;                                                    \
 182     (void)PartialMapSrcIdx;                                                    \
 183     const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize);               \
 184     (void)Map;                                                                 \
 185     assert(Map[0].BreakDown ==                                                 \
 186                &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] &&  \
 187            Map[0].NumBreakDowns == 1 && "FPR" #DstSize                         \
 188                                         " Dst is incorrectly initialized");    \
 189     assert(Map[1].BreakDown ==                                                 \
 190                &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] &&  \
 191            Map[1].NumBreakDowns == 1 && "FPR" #SrcSize                         \
 192                                         " Src is incorrectly initialized");    \
 193                                                                                \
 194   } while (false)
 195
 196   CHECK_VALUEMAP_FPEXT(32, 16);
 197   CHECK_VALUEMAP_FPEXT(64, 16);
 198   CHECK_VALUEMAP_FPEXT(64, 32);
 199   CHECK_VALUEMAP_FPEXT(128, 64);
 200
 201   assert(verify(TRI) && "Invalid register bank information");
 202 }
 203
 204 unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,
 205                                            const RegisterBank &B,
 206                                            unsigned Size) const {
 207   // What do we do with different size?
 208   // copy are same size.
 209   // Will introduce other hooks for different size:
 210   // * extract cost.
 211   // * build_sequence cost.
 212
 213   // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.
 214   // FIXME: This should be deduced from the scheduling model.
 215   if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)
 216     // FMOVXDr or FMOVWSr.
 217     return 5;
 218   if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)
 219     // FMOVDXr or FMOVSWr.
 220     return 4;
 221
 222   return RegisterBankInfo::copyCost(A, B, Size);
 223 }
 224
 225 const RegisterBank &AArch64RegisterBankInfo::getRegBankFromRegClass(
 226     const TargetRegisterClass &RC) const {
 227   switch (RC.getID()) {
 228   case AArch64::FPR8RegClassID:
 229   case AArch64::FPR16RegClassID:
 230   case AArch64::FPR32RegClassID:
 231   case AArch64::FPR64RegClassID:
 232   case AArch64::FPR128RegClassID:
 233   case AArch64::FPR128_loRegClassID:
 234   case AArch64::DDRegClassID:
 235   case AArch64::DDDRegClassID:
 236   case AArch64::DDDDRegClassID:
 237   case AArch64::QQRegClassID:
 238   case AArch64::QQQRegClassID:
 239   case AArch64::QQQQRegClassID:
 240     return getRegBank(AArch64::FPRRegBankID);
 241   case AArch64::GPR32commonRegClassID:
 242   case AArch64::GPR32RegClassID:
 243   case AArch64::GPR32spRegClassID:
 244   case AArch64::GPR32sponlyRegClassID:
 245   case AArch64::GPR32argRegClassID:
 246   case AArch64::GPR32allRegClassID:
 247   case AArch64::GPR64commonRegClassID:
 248   case AArch64::GPR64RegClassID:
 249   case AArch64::GPR64spRegClassID:
 250   case AArch64::GPR64sponlyRegClassID:
 251   case AArch64::GPR64argRegClassID:
 252   case AArch64::GPR64allRegClassID:
 253   case AArch64::GPR64noipRegClassID:
 254   case AArch64::GPR64common_and_GPR64noipRegClassID:
 255   case AArch64::GPR64noip_and_tcGPR64RegClassID:
 256   case AArch64::tcGPR64RegClassID:
 257   case AArch64::WSeqPairsClassRegClassID:
 258   case AArch64::XSeqPairsClassRegClassID:
 259     return getRegBank(AArch64::GPRRegBankID);
 260   case AArch64::CCRRegClassID:
 261     return getRegBank(AArch64::CCRegBankID);
 262   default:
 263     llvm_unreachable("Register class not supported");
 264   }
 265 }
 266
 267 RegisterBankInfo::InstructionMappings
 268 AArch64RegisterBankInfo::getInstrAlternativeMappings(
 269     const MachineInstr &MI) const {
 270   const MachineFunction &MF = *MI.getParent()->getParent();
 271   const TargetSubtargetInfo &STI = MF.getSubtarget();
 272   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
 273   const MachineRegisterInfo &MRI = MF.getRegInfo();
 274
 275   switch (MI.getOpcode()) {
 276   case TargetOpcode::G_OR: {
 277     // 32 and 64-bit or can be mapped on either FPR or
 278     // GPR for the same cost.
 279     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
 280     if (Size != 32 && Size != 64)
 281       break;
 282
 283     // If the instruction has any implicit-defs or uses,
 284     // do not mess with it.
 285     if (MI.getNumOperands() != 3)
 286       break;
 287     InstructionMappings AltMappings;
 288     const InstructionMapping &GPRMapping = getInstructionMapping(
 289         /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size),
 290         /*NumOperands*/ 3);
 291     const InstructionMapping &FPRMapping = getInstructionMapping(
 292         /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size),
 293         /*NumOperands*/ 3);
 294
 295     AltMappings.push_back(&GPRMapping);
 296     AltMappings.push_back(&FPRMapping);
 297     return AltMappings;
 298   }
 299   case TargetOpcode::G_BITCAST: {
 300     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
 301     if (Size != 32 && Size != 64)
 302       break;
 303
 304     // If the instruction has any implicit-defs or uses,
 305     // do not mess with it.
 306     if (MI.getNumOperands() != 2)
 307       break;
 308
 309     InstructionMappings AltMappings;
 310     const InstructionMapping &GPRMapping = getInstructionMapping(
 311         /*ID*/ 1, /*Cost*/ 1,
 312         getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size),
 313         /*NumOperands*/ 2);
 314     const InstructionMapping &FPRMapping = getInstructionMapping(
 315         /*ID*/ 2, /*Cost*/ 1,
 316         getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size),
 317         /*NumOperands*/ 2);
 318     const InstructionMapping &GPRToFPRMapping = getInstructionMapping(
 319         /*ID*/ 3,
 320         /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
 321         getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size),
 322         /*NumOperands*/ 2);
 323     const InstructionMapping &FPRToGPRMapping = getInstructionMapping(
 324         /*ID*/ 3,
 325         /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size),
 326         getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size),
 327         /*NumOperands*/ 2);
 328
 329     AltMappings.push_back(&GPRMapping);
 330     AltMappings.push_back(&FPRMapping);
 331     AltMappings.push_back(&GPRToFPRMapping);
 332     AltMappings.push_back(&FPRToGPRMapping);
 333     return AltMappings;
 334   }
 335   case TargetOpcode::G_LOAD: {
 336     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);
 337     if (Size != 64)
 338       break;
 339
 340     // If the instruction has any implicit-defs or uses,
 341     // do not mess with it.
 342     if (MI.getNumOperands() != 2)
 343       break;
 344
 345     InstructionMappings AltMappings;
 346     const InstructionMapping &GPRMapping = getInstructionMapping(
 347         /*ID*/ 1, /*Cost*/ 1,
 348         getOperandsMapping({getValueMapping(PMI_FirstGPR, Size),
 349                             // Addresses are GPR 64-bit.
 350                             getValueMapping(PMI_FirstGPR, 64)}),
 351         /*NumOperands*/ 2);
 352     const InstructionMapping &FPRMapping = getInstructionMapping(
 353         /*ID*/ 2, /*Cost*/ 1,
 354         getOperandsMapping({getValueMapping(PMI_FirstFPR, Size),
 355                             // Addresses are GPR 64-bit.
 356                             getValueMapping(PMI_FirstGPR, 64)}),
 357         /*NumOperands*/ 2);
 358
 359     AltMappings.push_back(&GPRMapping);
 360     AltMappings.push_back(&FPRMapping);
 361     return AltMappings;
 362   }
 363   default:
 364     break;
 365   }
 366   return RegisterBankInfo::getInstrAlternativeMappings(MI);
 367 }
 368
 369 void AArch64RegisterBankInfo::applyMappingImpl(
 370     const OperandsMapper &OpdMapper) const {
 371   switch (OpdMapper.getMI().getOpcode()) {
 372   case TargetOpcode::G_OR:
 373   case TargetOpcode::G_BITCAST:
 374   case TargetOpcode::G_LOAD:
 375     // Those ID must match getInstrAlternativeMappings.
 376     assert((OpdMapper.getInstrMapping().getID() >= 1 &&
 377             OpdMapper.getInstrMapping().getID() <= 4) &&
 378            "Don't know how to handle that ID");
 379     return applyDefaultMapping(OpdMapper);
 380   default:
 381     llvm_unreachable("Don't know how to handle that operation");
 382   }
 383 }
 384
 385 /// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
 386 /// having only floating-point operands.
 387 static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
 388   switch (Opc) {
 389   case TargetOpcode::G_FADD:
 390   case TargetOpcode::G_FSUB:
 391   case TargetOpcode::G_FMUL:
 392   case TargetOpcode::G_FDIV:
 393   case TargetOpcode::G_FCONSTANT:
 394   case TargetOpcode::G_FPEXT:
 395   case TargetOpcode::G_FPTRUNC:
 396   case TargetOpcode::G_FCEIL:
 397   case TargetOpcode::G_FFLOOR:
 398   case TargetOpcode::G_FNEG:
 399   case TargetOpcode::G_FCOS:
 400   case TargetOpcode::G_FSIN:
 401   case TargetOpcode::G_FLOG10:
 402   case TargetOpcode::G_FLOG:
 403   case TargetOpcode::G_FLOG2:
 404   case TargetOpcode::G_FSQRT:
 405   case TargetOpcode::G_FABS:
 406   case TargetOpcode::G_FEXP:
 407     return true;
 408   }
 409   return false;
 410 }
 411
 412 const RegisterBankInfo::InstructionMapping &
 413 AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
 414     const MachineInstr &MI) const {
 415   const unsigned Opc = MI.getOpcode();
 416   const MachineFunction &MF = *MI.getParent()->getParent();
 417   const MachineRegisterInfo &MRI = MF.getRegInfo();
 418
 419   unsigned NumOperands = MI.getNumOperands();
 420   assert(NumOperands <= 3 &&
 421          "This code is for instructions with 3 or less operands");
 422
 423   LLT Ty = MRI.getType(MI.getOperand(0).getReg());
 424   unsigned Size = Ty.getSizeInBits();
 425   bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
 426
 427   PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;
 428
 429 #ifndef NDEBUG
 430   // Make sure all the operands are using similar size and type.
 431   // Should probably be checked by the machine verifier.
 432   // This code won't catch cases where the number of lanes is
 433   // different between the operands.
 434   // If we want to go to that level of details, it is probably
 435   // best to check that the types are the same, period.
 436   // Currently, we just check that the register banks are the same
 437   // for each types.
 438   for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {
 439     LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());
 440     assert(
 441         AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(
 442             RBIdx, OpTy.getSizeInBits()) ==
 443             AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&
 444         "Operand has incompatible size");
 445     bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);
 446     (void)OpIsFPR;
 447     assert(IsFPR == OpIsFPR && "Operand has incompatible type");
 448   }
 449 #endif // End NDEBUG.
 450
 451   return getInstructionMapping(DefaultMappingID, 1,
 452                                getValueMapping(RBIdx, Size), NumOperands);
 453 }
 454
 455 const RegisterBankInfo::InstructionMapping &
 456 AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 457   const unsigned Opc = MI.getOpcode();
 458
 459   // Try the default logic for non-generic instructions that are either copies
 460   // or already have some operands assigned to banks.
 461   if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) ||
 462       Opc == TargetOpcode::G_PHI) {
 463     const RegisterBankInfo::InstructionMapping &Mapping =
 464         getInstrMappingImpl(MI);
 465     if (Mapping.isValid())
 466       return Mapping;
 467   }
 468
 469   const MachineFunction &MF = *MI.getParent()->getParent();
 470   const MachineRegisterInfo &MRI = MF.getRegInfo();
 471   const TargetSubtargetInfo &STI = MF.getSubtarget();
 472   const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
 473
 474   switch (Opc) {
 475     // G_{F|S|U}REM are not listed because they are not legal.
 476     // Arithmetic ops.
 477   case TargetOpcode::G_ADD:
 478   case TargetOpcode::G_SUB:
 479   case TargetOpcode::G_GEP:
 480   case TargetOpcode::G_MUL:
 481   case TargetOpcode::G_SDIV:
 482   case TargetOpcode::G_UDIV:
 483     // Bitwise ops.
 484   case TargetOpcode::G_AND:
 485   case TargetOpcode::G_OR:
 486   case TargetOpcode::G_XOR:
 487     // Shifts.
 488   case TargetOpcode::G_SHL:
 489   case TargetOpcode::G_LSHR:
 490   case TargetOpcode::G_ASHR:
 491     // Floating point ops.
 492   case TargetOpcode::G_FADD:
 493   case TargetOpcode::G_FSUB:
 494   case TargetOpcode::G_FMUL:
 495   case TargetOpcode::G_FDIV:
 496     return getSameKindOfOperandsMapping(MI);
 497   case TargetOpcode::G_FPEXT: {
 498     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
 499     LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
 500     return getInstructionMapping(
 501         DefaultMappingID, /*Cost*/ 1,
 502         getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()),
 503         /*NumOperands*/ 2);
 504   }
 505   case TargetOpcode::COPY: {
 506     unsigned DstReg = MI.getOperand(0).getReg();
 507     unsigned SrcReg = MI.getOperand(1).getReg();
 508     // Check if one of the register is not a generic register.
 509     if ((TargetRegisterInfo::isPhysicalRegister(DstReg) ||
 510          !MRI.getType(DstReg).isValid()) ||
 511         (TargetRegisterInfo::isPhysicalRegister(SrcReg) ||
 512          !MRI.getType(SrcReg).isValid())) {
 513       const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI);
 514       const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI);
 515       if (!DstRB)
 516         DstRB = SrcRB;
 517       else if (!SrcRB)
 518         SrcRB = DstRB;
 519       // If both RB are null that means both registers are generic.
 520       // We shouldn't be here.
 521       assert(DstRB && SrcRB && "Both RegBank were nullptr");
 522       unsigned Size = getSizeInBits(DstReg, MRI, TRI);
 523       return getInstructionMapping(
 524           DefaultMappingID, copyCost(*DstRB, *SrcRB, Size),
 525           getCopyMapping(DstRB->getID(), SrcRB->getID(), Size),
 526           // We only care about the mapping of the destination.
 527           /*NumOperands*/ 1);
 528     }
 529     // Both registers are generic, use G_BITCAST.
 530     LLVM_FALLTHROUGH;
 531   }
 532   case TargetOpcode::G_BITCAST: {
 533     LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
 534     LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
 535     unsigned Size = DstTy.getSizeInBits();
 536     bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;
 537     bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;
 538     const RegisterBank &DstRB =
 539         DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
 540     const RegisterBank &SrcRB =
 541         SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;
 542     return getInstructionMapping(
 543         DefaultMappingID, copyCost(DstRB, SrcRB, Size),
 544         getCopyMapping(DstRB.getID(), SrcRB.getID(), Size),
 545         // We only care about the mapping of the destination for COPY.
 546         /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);
 547   }
 548   default:
 549     break;
 550   }
 551
 552   unsigned NumOperands = MI.getNumOperands();
 553
 554   // Track the size and bank of each register.  We don't do partial mappings.
 555   SmallVector<unsigned, 4> OpSize(NumOperands);
 556   SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);
 557   for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
 558     auto &MO = MI.getOperand(Idx);
 559     if (!MO.isReg() || !MO.getReg())
 560       continue;
 561
 562     LLT Ty = MRI.getType(MO.getReg());
 563     OpSize[Idx] = Ty.getSizeInBits();
 564
 565     // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs.
 566     // For floating-point instructions, scalars go in FPRs.
 567     if (Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc) ||
 568         Ty.getSizeInBits() > 64)
 569       OpRegBankIdx[Idx] = PMI_FirstFPR;
 570     else
 571       OpRegBankIdx[Idx] = PMI_FirstGPR;
 572   }
 573
 574   unsigned Cost = 1;
 575   // Some of the floating-point instructions have mixed GPR and FPR operands:
 576   // fine-tune the computed mapping.
 577   switch (Opc) {
 578   case TargetOpcode::G_SITOFP:
 579   case TargetOpcode::G_UITOFP:
 580     if (MRI.getType(MI.getOperand(0).getReg()).isVector())
 581       break;
 582     OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
 583     break;
 584   case TargetOpcode::G_FPTOSI:
 585   case TargetOpcode::G_FPTOUI:
 586     if (MRI.getType(MI.getOperand(0).getReg()).isVector())
 587       break;
 588     OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
 589     break;
 590   case TargetOpcode::G_FCMP:
 591     OpRegBankIdx = {PMI_FirstGPR,
 592                     /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};
 593     break;
 594   case TargetOpcode::G_BITCAST:
 595     // This is going to be a cross register bank copy and this is expensive.
 596     if (OpRegBankIdx[0] != OpRegBankIdx[1])
 597       Cost = copyCost(
 598           *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,
 599           *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,
 600           OpSize[0]);
 601     break;
 602   case TargetOpcode::G_LOAD:
 603     // Loading in vector unit is slightly more expensive.
 604     // This is actually only true for the LD1R and co instructions,
 605     // but anyway for the fast mode this number does not matter and
 606     // for the greedy mode the cost of the cross bank copy will
 607     // offset this number.
 608     // FIXME: Should be derived from the scheduling model.
 609     if (OpRegBankIdx[0] != PMI_FirstGPR)
 610       Cost = 2;
 611     else
 612       // Check if that load feeds fp instructions.
 613       // In that case, we want the default mapping to be on FPR
 614       // instead of blind map every scalar to GPR.
 615       for (const MachineInstr &UseMI :
 616            MRI.use_instructions(MI.getOperand(0).getReg())) {
 617         // If we have at least one direct use in a FP instruction,
 618         // assume this was a floating point load in the IR.
 619         // If it was not, we would have had a bitcast before
 620         // reaching that instruction.
 621         unsigned UseOpc = UseMI.getOpcode();
 622         if (isPreISelGenericFloatingPointOpcode(UseOpc) ||
 623             // Check if we feed a copy-like instruction with
 624             // floating point constraints. In that case, we are still
 625             // feeding fp instructions, but indirectly
 626             // (e.g., through ABI copies).
 627             ((UseOpc == TargetOpcode::COPY || UseMI.isPHI()) &&
 628              getRegBank(UseMI.getOperand(0).getReg(), MRI, TRI) ==
 629                  &AArch64::FPRRegBank)) {
 630           OpRegBankIdx[0] = PMI_FirstFPR;
 631           break;
 632         }
 633       }
 634     break;
 635   case TargetOpcode::G_STORE:
 636     // Check if that store is fed by fp instructions.
 637     if (OpRegBankIdx[0] == PMI_FirstGPR) {
 638       unsigned VReg = MI.getOperand(0).getReg();
 639       if (!VReg)
 640         break;
 641       MachineInstr *DefMI = MRI.getVRegDef(VReg);
 642       unsigned DefOpc = DefMI->getOpcode();
 643       if (isPreISelGenericFloatingPointOpcode(DefOpc) ||
 644           // Check if we come from a copy-like instruction with
 645           // floating point constraints. In that case, we are still
 646           // fed by fp instructions, but indirectly
 647           // (e.g., through ABI copies).
 648           ((DefOpc == TargetOpcode::COPY || DefMI->isPHI()) &&
 649            getRegBank(DefMI->getOperand(0).getReg(), MRI, TRI) ==
 650                &AArch64::FPRRegBank))
 651         OpRegBankIdx[0] = PMI_FirstFPR;
 652       break;
 653     }
 654     break;
 655   case TargetOpcode::G_UNMERGE_VALUES: {
 656     // If the first operand belongs to a FPR register bank, then make sure that
 657     // we preserve that.
 658     if (OpRegBankIdx[0] != PMI_FirstGPR)
 659       break;
 660
 661     // Helper lambda that returns true if MI has floating point constraints.
 662     auto HasFPConstraints = [&TRI, &MRI, this](MachineInstr &MI) {
 663       unsigned Op = MI.getOpcode();
 664
 665       // Do we have an explicit floating point instruction?
 666       if (isPreISelGenericFloatingPointOpcode(Op))
 667         return true;
 668
 669       // No. Check if we have a copy-like instruction. If we do, then we could
 670       // still be fed by floating point instructions.
 671       if (Op != TargetOpcode::COPY && !MI.isPHI())
 672         return false;
 673
 674       // MI is copy-like. Return true if it's using an FPR.
 675       return getRegBank(MI.getOperand(0).getReg(), MRI, TRI) ==
 676              &AArch64::FPRRegBank;
 677     };
 678
 679     LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());
 680     // UNMERGE into scalars from a vector should always use FPR.
 681     // Likewise if any of the uses are FP instructions.
 682     if (SrcTy.isVector() ||
 683         any_of(MRI.use_instructions(MI.getOperand(0).getReg()),
 684                [&](MachineInstr &MI) { return HasFPConstraints(MI); })) {
 685       // Set the register bank of every operand to FPR.
 686       for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
 687            Idx < NumOperands; ++Idx)
 688         OpRegBankIdx[Idx] = PMI_FirstFPR;
 689     }
 690     break;
 691   }
 692
 693   case TargetOpcode::G_BUILD_VECTOR:
 694     // If the first source operand belongs to a FPR register bank, then make
 695     // sure that we preserve that.
 696     if (OpRegBankIdx[1] != PMI_FirstGPR)
 697       break;
 698     unsigned VReg = MI.getOperand(1).getReg();
 699     if (!VReg)
 700       break;
 701
 702     // Get the instruction that defined the source operand reg, and check if
 703     // it's a floating point operation.
 704     MachineInstr *DefMI = MRI.getVRegDef(VReg);
 705     unsigned DefOpc = DefMI->getOpcode();
 706     if (isPreISelGenericFloatingPointOpcode(DefOpc)) {
 707       // Have a floating point op.
 708       // Make sure every operand gets mapped to a FPR register class.
 709       unsigned NumOperands = MI.getNumOperands();
 710       for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
 711         OpRegBankIdx[Idx] = PMI_FirstFPR;
 712     }
 713     break;
 714   }
 715
 716   // Finally construct the computed mapping.
 717   SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);
 718   for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
 719     if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) {
 720       auto Mapping = getValueMapping(OpRegBankIdx[Idx], OpSize[Idx]);
 721       if (!Mapping->isValid())
 722         return getInvalidInstructionMapping();
 723
 724       OpdsMapping[Idx] = Mapping;
 725     }
 726   }
 727
 728   return getInstructionMapping(DefaultMappingID, Cost,
 729                                getOperandsMapping(OpdsMapping), NumOperands);
 730 }