lib/Target/X86/X86InstructionSelector.cpp

   1 //===- X86InstructionSelector.cpp -----------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 /// This file implements the targeting of the InstructionSelector class for
  10 /// X86.
  11 /// \todo This should be generated by TableGen.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "MCTargetDesc/X86BaseInfo.h"
  15 #include "X86InstrBuilder.h"
  16 #include "X86InstrInfo.h"
  17 #include "X86RegisterBankInfo.h"
  18 #include "X86RegisterInfo.h"
  19 #include "X86Subtarget.h"
  20 #include "X86TargetMachine.h"
  21 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  22 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
  23 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
  24 #include "llvm/CodeGen/GlobalISel/Utils.h"
  25 #include "llvm/CodeGen/MachineBasicBlock.h"
  26 #include "llvm/CodeGen/MachineConstantPool.h"
  27 #include "llvm/CodeGen/MachineFunction.h"
  28 #include "llvm/CodeGen/MachineInstr.h"
  29 #include "llvm/CodeGen/MachineInstrBuilder.h"
  30 #include "llvm/CodeGen/MachineMemOperand.h"
  31 #include "llvm/CodeGen/MachineOperand.h"
  32 #include "llvm/CodeGen/MachineRegisterInfo.h"
  33 #include "llvm/CodeGen/TargetOpcodes.h"
  34 #include "llvm/CodeGen/TargetRegisterInfo.h"
  35 #include "llvm/IR/DataLayout.h"
  36 #include "llvm/IR/InstrTypes.h"
  37 #include "llvm/Support/AtomicOrdering.h"
  38 #include "llvm/Support/CodeGen.h"
  39 #include "llvm/Support/Debug.h"
  40 #include "llvm/Support/ErrorHandling.h"
  41 #include "llvm/Support/LowLevelTypeImpl.h"
  42 #include "llvm/Support/MathExtras.h"
  43 #include "llvm/Support/raw_ostream.h"
  44 #include <cassert>
  45 #include <cstdint>
  46 #include <tuple>
  47
  48 #define DEBUG_TYPE "X86-isel"
  49
  50 using namespace llvm;
  51
  52 namespace {
  53
  54 #define GET_GLOBALISEL_PREDICATE_BITSET
  55 #include "X86GenGlobalISel.inc"
  56 #undef GET_GLOBALISEL_PREDICATE_BITSET
  57
  58 class X86InstructionSelector : public InstructionSelector {
  59 public:
  60   X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
  61                          const X86RegisterBankInfo &RBI);
  62
  63   bool select(MachineInstr &I) override;
  64   static const char *getName() { return DEBUG_TYPE; }
  65
  66 private:
  67   /// tblgen-erated 'select' implementation, used as the initial selector for
  68   /// the patterns that don't require complex C++.
  69   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
  70
  71   // TODO: remove after supported by Tablegen-erated instruction selection.
  72   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
  73                           uint64_t Alignment) const;
  74
  75   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
  76                          MachineFunction &MF) const;
  77   bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
  78                              MachineFunction &MF) const;
  79   bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
  80                          MachineFunction &MF) const;
  81   bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
  82                       MachineFunction &MF) const;
  83   bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
  84                              MachineFunction &MF) const;
  85   bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
  86                   MachineFunction &MF) const;
  87   bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
  88                     MachineFunction &MF) const;
  89   bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
  90                  MachineFunction &MF) const;
  91   bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
  92                   MachineFunction &MF) const;
  93   bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
  94                    MachineFunction &MF) const;
  95   bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
  96   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
  97                            MachineFunction &MF);
  98   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
  99                          MachineFunction &MF);
 100   bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
 101                     MachineFunction &MF) const;
 102   bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
 103                      MachineFunction &MF) const;
 104   bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
 105                         MachineFunction &MF) const;
 106   bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
 107                           const unsigned DstReg,
 108                           const TargetRegisterClass *DstRC,
 109                           const unsigned SrcReg,
 110                           const TargetRegisterClass *SrcRC) const;
 111   bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
 112                      MachineFunction &MF) const;
 113   bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
 114   bool selectShift(MachineInstr &I, MachineRegisterInfo &MRI,
 115                    MachineFunction &MF) const;
 116   bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
 117                     MachineFunction &MF) const;
 118   bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
 119                                    MachineFunction &MF) const;
 120
 121   // emit insert subreg instruction and insert it before MachineInstr &I
 122   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
 123                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
 124   // emit extract subreg instruction and insert it before MachineInstr &I
 125   bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
 126                          MachineRegisterInfo &MRI, MachineFunction &MF) const;
 127
 128   const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
 129   const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
 130                                          MachineRegisterInfo &MRI) const;
 131
 132   const X86TargetMachine &TM;
 133   const X86Subtarget &STI;
 134   const X86InstrInfo &TII;
 135   const X86RegisterInfo &TRI;
 136   const X86RegisterBankInfo &RBI;
 137
 138 #define GET_GLOBALISEL_PREDICATES_DECL
 139 #include "X86GenGlobalISel.inc"
 140 #undef GET_GLOBALISEL_PREDICATES_DECL
 141
 142 #define GET_GLOBALISEL_TEMPORARIES_DECL
 143 #include "X86GenGlobalISel.inc"
 144 #undef GET_GLOBALISEL_TEMPORARIES_DECL
 145 };
 146
 147 } // end anonymous namespace
 148
 149 #define GET_GLOBALISEL_IMPL
 150 #include "X86GenGlobalISel.inc"
 151 #undef GET_GLOBALISEL_IMPL
 152
 153 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
 154                                                const X86Subtarget &STI,
 155                                                const X86RegisterBankInfo &RBI)
 156     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
 157       TRI(*STI.getRegisterInfo()), RBI(RBI),
 158 #define GET_GLOBALISEL_PREDICATES_INIT
 159 #include "X86GenGlobalISel.inc"
 160 #undef GET_GLOBALISEL_PREDICATES_INIT
 161 #define GET_GLOBALISEL_TEMPORARIES_INIT
 162 #include "X86GenGlobalISel.inc"
 163 #undef GET_GLOBALISEL_TEMPORARIES_INIT
 164 {
 165 }
 166
 167 // FIXME: This should be target-independent, inferred from the types declared
 168 // for each class in the bank.
 169 const TargetRegisterClass *
 170 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
 171   if (RB.getID() == X86::GPRRegBankID) {
 172     if (Ty.getSizeInBits() <= 8)
 173       return &X86::GR8RegClass;
 174     if (Ty.getSizeInBits() == 16)
 175       return &X86::GR16RegClass;
 176     if (Ty.getSizeInBits() == 32)
 177       return &X86::GR32RegClass;
 178     if (Ty.getSizeInBits() == 64)
 179       return &X86::GR64RegClass;
 180   }
 181   if (RB.getID() == X86::VECRRegBankID) {
 182     if (Ty.getSizeInBits() == 32)
 183       return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
 184     if (Ty.getSizeInBits() == 64)
 185       return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
 186     if (Ty.getSizeInBits() == 128)
 187       return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
 188     if (Ty.getSizeInBits() == 256)
 189       return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
 190     if (Ty.getSizeInBits() == 512)
 191       return &X86::VR512RegClass;
 192   }
 193
 194   llvm_unreachable("Unknown RegBank!");
 195 }
 196
 197 const TargetRegisterClass *
 198 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
 199                                     MachineRegisterInfo &MRI) const {
 200   const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
 201   return getRegClass(Ty, RegBank);
 202 }
 203
 204 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
 205   unsigned SubIdx = X86::NoSubRegister;
 206   if (RC == &X86::GR32RegClass) {
 207     SubIdx = X86::sub_32bit;
 208   } else if (RC == &X86::GR16RegClass) {
 209     SubIdx = X86::sub_16bit;
 210   } else if (RC == &X86::GR8RegClass) {
 211     SubIdx = X86::sub_8bit;
 212   }
 213
 214   return SubIdx;
 215 }
 216
 217 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
 218   assert(Register::isPhysicalRegister(Reg));
 219   if (X86::GR64RegClass.contains(Reg))
 220     return &X86::GR64RegClass;
 221   if (X86::GR32RegClass.contains(Reg))
 222     return &X86::GR32RegClass;
 223   if (X86::GR16RegClass.contains(Reg))
 224     return &X86::GR16RegClass;
 225   if (X86::GR8RegClass.contains(Reg))
 226     return &X86::GR8RegClass;
 227
 228   llvm_unreachable("Unknown RegClass for PhysReg!");
 229 }
 230
 231 // Set X86 Opcode and constrain DestReg.
 232 bool X86InstructionSelector::selectCopy(MachineInstr &I,
 233                                         MachineRegisterInfo &MRI) const {
 234   Register DstReg = I.getOperand(0).getReg();
 235   const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
 236   const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
 237
 238   Register SrcReg = I.getOperand(1).getReg();
 239   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
 240   const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
 241
 242   if (Register::isPhysicalRegister(DstReg)) {
 243     assert(I.isCopy() && "Generic operators do not allow physical registers");
 244
 245     if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
 246         DstRegBank.getID() == X86::GPRRegBankID) {
 247
 248       const TargetRegisterClass *SrcRC =
 249           getRegClass(MRI.getType(SrcReg), SrcRegBank);
 250       const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
 251
 252       if (SrcRC != DstRC) {
 253         // This case can be generated by ABI lowering, performe anyext
 254         Register ExtSrc = MRI.createVirtualRegister(DstRC);
 255         BuildMI(*I.getParent(), I, I.getDebugLoc(),
 256                 TII.get(TargetOpcode::SUBREG_TO_REG))
 257             .addDef(ExtSrc)
 258             .addImm(0)
 259             .addReg(SrcReg)
 260             .addImm(getSubRegIndex(SrcRC));
 261
 262         I.getOperand(1).setReg(ExtSrc);
 263       }
 264     }
 265
 266     return true;
 267   }
 268
 269   assert((!Register::isPhysicalRegister(SrcReg) || I.isCopy()) &&
 270          "No phys reg on generic operators");
 271   assert((DstSize == SrcSize ||
 272           // Copies are a mean to setup initial types, the number of
 273           // bits may not exactly match.
 274           (Register::isPhysicalRegister(SrcReg) &&
 275            DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
 276          "Copy with different width?!");
 277
 278   const TargetRegisterClass *DstRC =
 279       getRegClass(MRI.getType(DstReg), DstRegBank);
 280
 281   if (SrcRegBank.getID() == X86::GPRRegBankID &&
 282       DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
 283       Register::isPhysicalRegister(SrcReg)) {
 284     // Change the physical register to performe truncate.
 285
 286     const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
 287
 288     if (DstRC != SrcRC) {
 289       I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
 290       I.getOperand(1).substPhysReg(SrcReg, TRI);
 291     }
 292   }
 293
 294   // No need to constrain SrcReg. It will get constrained when
 295   // we hit another of its use or its defs.
 296   // Copies do not have constraints.
 297   const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
 298   if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
 299     if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
 300       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
 301                         << " operand\n");
 302       return false;
 303     }
 304   }
 305   I.setDesc(TII.get(X86::COPY));
 306   return true;
 307 }
 308
 309 bool X86InstructionSelector::select(MachineInstr &I) {
 310   assert(I.getParent() && "Instruction should be in a basic block!");
 311   assert(I.getParent()->getParent() && "Instruction should be in a function!");
 312
 313   MachineBasicBlock &MBB = *I.getParent();
 314   MachineFunction &MF = *MBB.getParent();
 315   MachineRegisterInfo &MRI = MF.getRegInfo();
 316
 317   unsigned Opcode = I.getOpcode();
 318   if (!isPreISelGenericOpcode(Opcode)) {
 319     // Certain non-generic instructions also need some special handling.
 320
 321     if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
 322       return false;
 323
 324     if (I.isCopy())
 325       return selectCopy(I, MRI);
 326
 327     return true;
 328   }
 329
 330   assert(I.getNumOperands() == I.getNumExplicitOperands() &&
 331          "Generic instruction has unexpected implicit operands\n");
 332
 333   if (selectImpl(I, *CoverageInfo))
 334     return true;
 335
 336   LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
 337
 338   // TODO: This should be implemented by tblgen.
 339   switch (I.getOpcode()) {
 340   default:
 341     return false;
 342   case TargetOpcode::G_STORE:
 343   case TargetOpcode::G_LOAD:
 344     return selectLoadStoreOp(I, MRI, MF);
 345   case TargetOpcode::G_GEP:
 346   case TargetOpcode::G_FRAME_INDEX:
 347     return selectFrameIndexOrGep(I, MRI, MF);
 348   case TargetOpcode::G_GLOBAL_VALUE:
 349     return selectGlobalValue(I, MRI, MF);
 350   case TargetOpcode::G_CONSTANT:
 351     return selectConstant(I, MRI, MF);
 352   case TargetOpcode::G_FCONSTANT:
 353     return materializeFP(I, MRI, MF);
 354   case TargetOpcode::G_PTRTOINT:
 355   case TargetOpcode::G_TRUNC:
 356     return selectTruncOrPtrToInt(I, MRI, MF);
 357   case TargetOpcode::G_INTTOPTR:
 358     return selectCopy(I, MRI);
 359   case TargetOpcode::G_ZEXT:
 360     return selectZext(I, MRI, MF);
 361   case TargetOpcode::G_ANYEXT:
 362     return selectAnyext(I, MRI, MF);
 363   case TargetOpcode::G_ICMP:
 364     return selectCmp(I, MRI, MF);
 365   case TargetOpcode::G_FCMP:
 366     return selectFCmp(I, MRI, MF);
 367   case TargetOpcode::G_UADDE:
 368     return selectUadde(I, MRI, MF);
 369   case TargetOpcode::G_UNMERGE_VALUES:
 370     return selectUnmergeValues(I, MRI, MF);
 371   case TargetOpcode::G_MERGE_VALUES:
 372   case TargetOpcode::G_CONCAT_VECTORS:
 373     return selectMergeValues(I, MRI, MF);
 374   case TargetOpcode::G_EXTRACT:
 375     return selectExtract(I, MRI, MF);
 376   case TargetOpcode::G_INSERT:
 377     return selectInsert(I, MRI, MF);
 378   case TargetOpcode::G_BRCOND:
 379     return selectCondBranch(I, MRI, MF);
 380   case TargetOpcode::G_IMPLICIT_DEF:
 381   case TargetOpcode::G_PHI:
 382     return selectImplicitDefOrPHI(I, MRI);
 383   case TargetOpcode::G_SHL:
 384   case TargetOpcode::G_ASHR:
 385   case TargetOpcode::G_LSHR:
 386     return selectShift(I, MRI, MF);
 387   case TargetOpcode::G_SDIV:
 388   case TargetOpcode::G_UDIV:
 389   case TargetOpcode::G_SREM:
 390   case TargetOpcode::G_UREM:
 391     return selectDivRem(I, MRI, MF);
 392   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
 393     return selectIntrinsicWSideEffects(I, MRI, MF);
 394   }
 395
 396   return false;
 397 }
 398
 399 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
 400                                                 const RegisterBank &RB,
 401                                                 unsigned Opc,
 402                                                 uint64_t Alignment) const {
 403   bool Isload = (Opc == TargetOpcode::G_LOAD);
 404   bool HasAVX = STI.hasAVX();
 405   bool HasAVX512 = STI.hasAVX512();
 406   bool HasVLX = STI.hasVLX();
 407
 408   if (Ty == LLT::scalar(8)) {
 409     if (X86::GPRRegBankID == RB.getID())
 410       return Isload ? X86::MOV8rm : X86::MOV8mr;
 411   } else if (Ty == LLT::scalar(16)) {
 412     if (X86::GPRRegBankID == RB.getID())
 413       return Isload ? X86::MOV16rm : X86::MOV16mr;
 414   } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
 415     if (X86::GPRRegBankID == RB.getID())
 416       return Isload ? X86::MOV32rm : X86::MOV32mr;
 417     if (X86::VECRRegBankID == RB.getID())
 418       return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
 419                        HasAVX    ? X86::VMOVSSrm_alt :
 420                                    X86::MOVSSrm_alt)
 421                     : (HasAVX512 ? X86::VMOVSSZmr :
 422                        HasAVX    ? X86::VMOVSSmr :
 423                                    X86::MOVSSmr);
 424   } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
 425     if (X86::GPRRegBankID == RB.getID())
 426       return Isload ? X86::MOV64rm : X86::MOV64mr;
 427     if (X86::VECRRegBankID == RB.getID())
 428       return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
 429                        HasAVX    ? X86::VMOVSDrm_alt :
 430                                    X86::MOVSDrm_alt)
 431                     : (HasAVX512 ? X86::VMOVSDZmr :
 432                        HasAVX    ? X86::VMOVSDmr :
 433                                    X86::MOVSDmr);
 434   } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
 435     if (Alignment >= 16)
 436       return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
 437                               : HasAVX512
 438                                     ? X86::VMOVAPSZ128rm_NOVLX
 439                                     : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
 440                     : (HasVLX ? X86::VMOVAPSZ128mr
 441                               : HasAVX512
 442                                     ? X86::VMOVAPSZ128mr_NOVLX
 443                                     : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
 444     else
 445       return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
 446                               : HasAVX512
 447                                     ? X86::VMOVUPSZ128rm_NOVLX
 448                                     : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
 449                     : (HasVLX ? X86::VMOVUPSZ128mr
 450                               : HasAVX512
 451                                     ? X86::VMOVUPSZ128mr_NOVLX
 452                                     : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
 453   } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
 454     if (Alignment >= 32)
 455       return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
 456                               : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
 457                                           : X86::VMOVAPSYrm)
 458                     : (HasVLX ? X86::VMOVAPSZ256mr
 459                               : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
 460                                           : X86::VMOVAPSYmr);
 461     else
 462       return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
 463                               : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
 464                                           : X86::VMOVUPSYrm)
 465                     : (HasVLX ? X86::VMOVUPSZ256mr
 466                               : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
 467                                           : X86::VMOVUPSYmr);
 468   } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
 469     if (Alignment >= 64)
 470       return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
 471     else
 472       return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
 473   }
 474   return Opc;
 475 }
 476
 477 // Fill in an address from the given instruction.
 478 static void X86SelectAddress(const MachineInstr &I,
 479                              const MachineRegisterInfo &MRI,
 480                              X86AddressMode &AM) {
 481   assert(I.getOperand(0).isReg() && "unsupported opperand.");
 482   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
 483          "unsupported type.");
 484
 485   if (I.getOpcode() == TargetOpcode::G_GEP) {
 486     if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
 487       int64_t Imm = *COff;
 488       if (isInt<32>(Imm)) { // Check for displacement overflow.
 489         AM.Disp = static_cast<int32_t>(Imm);
 490         AM.Base.Reg = I.getOperand(1).getReg();
 491         return;
 492       }
 493     }
 494   } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
 495     AM.Base.FrameIndex = I.getOperand(1).getIndex();
 496     AM.BaseType = X86AddressMode::FrameIndexBase;
 497     return;
 498   }
 499
 500   // Default behavior.
 501   AM.Base.Reg = I.getOperand(0).getReg();
 502 }
 503
 504 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
 505                                                MachineRegisterInfo &MRI,
 506                                                MachineFunction &MF) const {
 507   unsigned Opc = I.getOpcode();
 508
 509   assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
 510          "unexpected instruction");
 511
 512   const Register DefReg = I.getOperand(0).getReg();
 513   LLT Ty = MRI.getType(DefReg);
 514   const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
 515
 516   assert(I.hasOneMemOperand());
 517   auto &MemOp = **I.memoperands_begin();
 518   if (MemOp.isAtomic()) {
 519     // Note: for unordered operations, we rely on the fact the appropriate MMO
 520     // is already on the instruction we're mutating, and thus we don't need to
 521     // make any changes.  So long as we select an opcode which is capable of
 522     // loading or storing the appropriate size atomically, the rest of the
 523     // backend is required to respect the MMO state.
 524     if (!MemOp.isUnordered()) {
 525       LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
 526       return false;
 527     }
 528     if (MemOp.getAlignment() < Ty.getSizeInBits()/8) {
 529       LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
 530       return false;
 531     }
 532   }
 533
 534   unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
 535   if (NewOpc == Opc)
 536     return false;
 537
 538   X86AddressMode AM;
 539   X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
 540
 541   I.setDesc(TII.get(NewOpc));
 542   MachineInstrBuilder MIB(MF, I);
 543   if (Opc == TargetOpcode::G_LOAD) {
 544     I.RemoveOperand(1);
 545     addFullAddress(MIB, AM);
 546   } else {
 547     // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
 548     I.RemoveOperand(1);
 549     I.RemoveOperand(0);
 550     addFullAddress(MIB, AM).addUse(DefReg);
 551   }
 552   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 553 }
 554
 555 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
 556   if (Ty == LLT::pointer(0, 64))
 557     return X86::LEA64r;
 558   else if (Ty == LLT::pointer(0, 32))
 559     return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
 560   else
 561     llvm_unreachable("Can't get LEA opcode. Unsupported type.");
 562 }
 563
 564 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
 565                                                    MachineRegisterInfo &MRI,
 566                                                    MachineFunction &MF) const {
 567   unsigned Opc = I.getOpcode();
 568
 569   assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_GEP) &&
 570          "unexpected instruction");
 571
 572   const Register DefReg = I.getOperand(0).getReg();
 573   LLT Ty = MRI.getType(DefReg);
 574
 575   // Use LEA to calculate frame index and GEP
 576   unsigned NewOpc = getLeaOP(Ty, STI);
 577   I.setDesc(TII.get(NewOpc));
 578   MachineInstrBuilder MIB(MF, I);
 579
 580   if (Opc == TargetOpcode::G_FRAME_INDEX) {
 581     addOffset(MIB, 0);
 582   } else {
 583     MachineOperand &InxOp = I.getOperand(2);
 584     I.addOperand(InxOp);        // set IndexReg
 585     InxOp.ChangeToImmediate(1); // set Scale
 586     MIB.addImm(0).addReg(0);
 587   }
 588
 589   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 590 }
 591
 592 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
 593                                                MachineRegisterInfo &MRI,
 594                                                MachineFunction &MF) const {
 595   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
 596          "unexpected instruction");
 597
 598   auto GV = I.getOperand(1).getGlobal();
 599   if (GV->isThreadLocal()) {
 600     return false; // TODO: we don't support TLS yet.
 601   }
 602
 603   // Can't handle alternate code models yet.
 604   if (TM.getCodeModel() != CodeModel::Small)
 605     return false;
 606
 607   X86AddressMode AM;
 608   AM.GV = GV;
 609   AM.GVOpFlags = STI.classifyGlobalReference(GV);
 610
 611   // TODO: The ABI requires an extra load. not supported yet.
 612   if (isGlobalStubReference(AM.GVOpFlags))
 613     return false;
 614
 615   // TODO: This reference is relative to the pic base. not supported yet.
 616   if (isGlobalRelativeToPICBase(AM.GVOpFlags))
 617     return false;
 618
 619   if (STI.isPICStyleRIPRel()) {
 620     // Use rip-relative addressing.
 621     assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
 622     AM.Base.Reg = X86::RIP;
 623   }
 624
 625   const Register DefReg = I.getOperand(0).getReg();
 626   LLT Ty = MRI.getType(DefReg);
 627   unsigned NewOpc = getLeaOP(Ty, STI);
 628
 629   I.setDesc(TII.get(NewOpc));
 630   MachineInstrBuilder MIB(MF, I);
 631
 632   I.RemoveOperand(1);
 633   addFullAddress(MIB, AM);
 634
 635   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 636 }
 637
 638 bool X86InstructionSelector::selectConstant(MachineInstr &I,
 639                                             MachineRegisterInfo &MRI,
 640                                             MachineFunction &MF) const {
 641   assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
 642          "unexpected instruction");
 643
 644   const Register DefReg = I.getOperand(0).getReg();
 645   LLT Ty = MRI.getType(DefReg);
 646
 647   if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
 648     return false;
 649
 650   uint64_t Val = 0;
 651   if (I.getOperand(1).isCImm()) {
 652     Val = I.getOperand(1).getCImm()->getZExtValue();
 653     I.getOperand(1).ChangeToImmediate(Val);
 654   } else if (I.getOperand(1).isImm()) {
 655     Val = I.getOperand(1).getImm();
 656   } else
 657     llvm_unreachable("Unsupported operand type.");
 658
 659   unsigned NewOpc;
 660   switch (Ty.getSizeInBits()) {
 661   case 8:
 662     NewOpc = X86::MOV8ri;
 663     break;
 664   case 16:
 665     NewOpc = X86::MOV16ri;
 666     break;
 667   case 32:
 668     NewOpc = X86::MOV32ri;
 669     break;
 670   case 64:
 671     // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
 672     if (isInt<32>(Val))
 673       NewOpc = X86::MOV64ri32;
 674     else
 675       NewOpc = X86::MOV64ri;
 676     break;
 677   default:
 678     llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
 679   }
 680
 681   I.setDesc(TII.get(NewOpc));
 682   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
 683 }
 684
 685 // Helper function for selectTruncOrPtrToInt and selectAnyext.
 686 // Returns true if DstRC lives on a floating register class and
 687 // SrcRC lives on a 128-bit vector class.
 688 static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
 689                             const TargetRegisterClass *SrcRC) {
 690   return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
 691           DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
 692          (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
 693 }
 694
 695 bool X86InstructionSelector::selectTurnIntoCOPY(
 696     MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
 697     const TargetRegisterClass *DstRC, const unsigned SrcReg,
 698     const TargetRegisterClass *SrcRC) const {
 699
 700   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
 701       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
 702     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
 703                       << " operand\n");
 704     return false;
 705   }
 706   I.setDesc(TII.get(X86::COPY));
 707   return true;
 708 }
 709
 710 bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
 711                                                    MachineRegisterInfo &MRI,
 712                                                    MachineFunction &MF) const {
 713   assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
 714           I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
 715          "unexpected instruction");
 716
 717   const Register DstReg = I.getOperand(0).getReg();
 718   const Register SrcReg = I.getOperand(1).getReg();
 719
 720   const LLT DstTy = MRI.getType(DstReg);
 721   const LLT SrcTy = MRI.getType(SrcReg);
 722
 723   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
 724   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
 725
 726   if (DstRB.getID() != SrcRB.getID()) {
 727     LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
 728                       << " input/output on different banks\n");
 729     return false;
 730   }
 731
 732   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
 733   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
 734
 735   if (!DstRC || !SrcRC)
 736     return false;
 737
 738   // If that's truncation of the value that lives on the vector class and goes
 739   // into the floating class, just replace it with copy, as we are able to
 740   // select it as a regular move.
 741   if (canTurnIntoCOPY(DstRC, SrcRC))
 742     return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
 743
 744   if (DstRB.getID() != X86::GPRRegBankID)
 745     return false;
 746
 747   unsigned SubIdx;
 748   if (DstRC == SrcRC) {
 749     // Nothing to be done
 750     SubIdx = X86::NoSubRegister;
 751   } else if (DstRC == &X86::GR32RegClass) {
 752     SubIdx = X86::sub_32bit;
 753   } else if (DstRC == &X86::GR16RegClass) {
 754     SubIdx = X86::sub_16bit;
 755   } else if (DstRC == &X86::GR8RegClass) {
 756     SubIdx = X86::sub_8bit;
 757   } else {
 758     return false;
 759   }
 760
 761   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
 762
 763   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
 764       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
 765     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
 766                       << "\n");
 767     return false;
 768   }
 769
 770   I.getOperand(1).setSubReg(SubIdx);
 771
 772   I.setDesc(TII.get(X86::COPY));
 773   return true;
 774 }
 775
 776 bool X86InstructionSelector::selectZext(MachineInstr &I,
 777                                         MachineRegisterInfo &MRI,
 778                                         MachineFunction &MF) const {
 779   assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
 780
 781   const Register DstReg = I.getOperand(0).getReg();
 782   const Register SrcReg = I.getOperand(1).getReg();
 783
 784   const LLT DstTy = MRI.getType(DstReg);
 785   const LLT SrcTy = MRI.getType(SrcReg);
 786
 787   assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
 788          "8=>32 Zext is handled by tablegen");
 789   assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
 790          "16=>32 Zext is handled by tablegen");
 791
 792   const static struct ZextEntry {
 793     LLT SrcTy;
 794     LLT DstTy;
 795     unsigned MovOp;
 796     bool NeedSubregToReg;
 797   } OpTable[] = {
 798       {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false},  // i8  => i16
 799       {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true},   // i8  => i64
 800       {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
 801       {LLT::scalar(32), LLT::scalar(64), 0, true}                 // i32 => i64
 802   };
 803
 804   auto ZextEntryIt =
 805       std::find_if(std::begin(OpTable), std::end(OpTable),
 806                    [SrcTy, DstTy](const ZextEntry &El) {
 807                      return El.DstTy == DstTy && El.SrcTy == SrcTy;
 808                    });
 809
 810   // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
 811   if (ZextEntryIt != std::end(OpTable)) {
 812     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
 813     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
 814     const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
 815     const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
 816
 817     if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
 818         !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
 819       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
 820                         << " operand\n");
 821       return false;
 822     }
 823
 824     unsigned TransitRegTo = DstReg;
 825     unsigned TransitRegFrom = SrcReg;
 826     if (ZextEntryIt->MovOp) {
 827       // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
 828       // a transit register in between: create it here.
 829       if (ZextEntryIt->NeedSubregToReg) {
 830         TransitRegFrom = MRI.createVirtualRegister(
 831             getRegClass(LLT::scalar(32), DstReg, MRI));
 832         TransitRegTo = TransitRegFrom;
 833       }
 834
 835       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
 836           .addDef(TransitRegTo)
 837           .addReg(SrcReg);
 838     }
 839     if (ZextEntryIt->NeedSubregToReg) {
 840       BuildMI(*I.getParent(), I, I.getDebugLoc(),
 841               TII.get(TargetOpcode::SUBREG_TO_REG))
 842           .addDef(DstReg)
 843           .addImm(0)
 844           .addReg(TransitRegFrom)
 845           .addImm(X86::sub_32bit);
 846     }
 847     I.eraseFromParent();
 848     return true;
 849   }
 850
 851   if (SrcTy != LLT::scalar(1))
 852     return false;
 853
 854   unsigned AndOpc;
 855   if (DstTy == LLT::scalar(8))
 856     AndOpc = X86::AND8ri;
 857   else if (DstTy == LLT::scalar(16))
 858     AndOpc = X86::AND16ri8;
 859   else if (DstTy == LLT::scalar(32))
 860     AndOpc = X86::AND32ri8;
 861   else if (DstTy == LLT::scalar(64))
 862     AndOpc = X86::AND64ri8;
 863   else
 864     return false;
 865
 866   unsigned DefReg = SrcReg;
 867   if (DstTy != LLT::scalar(8)) {
 868     DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
 869     BuildMI(*I.getParent(), I, I.getDebugLoc(),
 870             TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
 871         .addImm(0)
 872         .addReg(SrcReg)
 873         .addImm(X86::sub_8bit);
 874   }
 875
 876   MachineInstr &AndInst =
 877       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
 878            .addReg(DefReg)
 879            .addImm(1);
 880
 881   constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
 882
 883   I.eraseFromParent();
 884   return true;
 885 }
 886
 887 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
 888                                           MachineRegisterInfo &MRI,
 889                                           MachineFunction &MF) const {
 890   assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
 891
 892   const Register DstReg = I.getOperand(0).getReg();
 893   const Register SrcReg = I.getOperand(1).getReg();
 894
 895   const LLT DstTy = MRI.getType(DstReg);
 896   const LLT SrcTy = MRI.getType(SrcReg);
 897
 898   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
 899   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
 900
 901   assert(DstRB.getID() == SrcRB.getID() &&
 902          "G_ANYEXT input/output on different banks\n");
 903
 904   assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
 905          "G_ANYEXT incorrect operand size");
 906
 907   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
 908   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
 909
 910   // If that's ANY_EXT of the value that lives on the floating class and goes
 911   // into the vector class, just replace it with copy, as we are able to select
 912   // it as a regular move.
 913   if (canTurnIntoCOPY(SrcRC, DstRC))
 914     return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
 915
 916   if (DstRB.getID() != X86::GPRRegBankID)
 917     return false;
 918
 919   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
 920       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
 921     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
 922                       << " operand\n");
 923     return false;
 924   }
 925
 926   if (SrcRC == DstRC) {
 927     I.setDesc(TII.get(X86::COPY));
 928     return true;
 929   }
 930
 931   BuildMI(*I.getParent(), I, I.getDebugLoc(),
 932           TII.get(TargetOpcode::SUBREG_TO_REG))
 933       .addDef(DstReg)
 934       .addImm(0)
 935       .addReg(SrcReg)
 936       .addImm(getSubRegIndex(SrcRC));
 937
 938   I.eraseFromParent();
 939   return true;
 940 }
 941
 942 bool X86InstructionSelector::selectCmp(MachineInstr &I,
 943                                        MachineRegisterInfo &MRI,
 944                                        MachineFunction &MF) const {
 945   assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
 946
 947   X86::CondCode CC;
 948   bool SwapArgs;
 949   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
 950       (CmpInst::Predicate)I.getOperand(1).getPredicate());
 951
 952   Register LHS = I.getOperand(2).getReg();
 953   Register RHS = I.getOperand(3).getReg();
 954
 955   if (SwapArgs)
 956     std::swap(LHS, RHS);
 957
 958   unsigned OpCmp;
 959   LLT Ty = MRI.getType(LHS);
 960
 961   switch (Ty.getSizeInBits()) {
 962   default:
 963     return false;
 964   case 8:
 965     OpCmp = X86::CMP8rr;
 966     break;
 967   case 16:
 968     OpCmp = X86::CMP16rr;
 969     break;
 970   case 32:
 971     OpCmp = X86::CMP32rr;
 972     break;
 973   case 64:
 974     OpCmp = X86::CMP64rr;
 975     break;
 976   }
 977
 978   MachineInstr &CmpInst =
 979       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
 980            .addReg(LHS)
 981            .addReg(RHS);
 982
 983   MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
 984                                    TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
 985
 986   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
 987   constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
 988
 989   I.eraseFromParent();
 990   return true;
 991 }
 992
 993 bool X86InstructionSelector::selectFCmp(MachineInstr &I,
 994                                         MachineRegisterInfo &MRI,
 995                                         MachineFunction &MF) const {
 996   assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
 997
 998   Register LhsReg = I.getOperand(2).getReg();
 999   Register RhsReg = I.getOperand(3).getReg();
1000   CmpInst::Predicate Predicate =
1001       (CmpInst::Predicate)I.getOperand(1).getPredicate();
1002
1003   // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1004   static const uint16_t SETFOpcTable[2][3] = {
1005       {X86::COND_E, X86::COND_NP, X86::AND8rr},
1006       {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1007   const uint16_t *SETFOpc = nullptr;
1008   switch (Predicate) {
1009   default:
1010     break;
1011   case CmpInst::FCMP_OEQ:
1012     SETFOpc = &SETFOpcTable[0][0];
1013     break;
1014   case CmpInst::FCMP_UNE:
1015     SETFOpc = &SETFOpcTable[1][0];
1016     break;
1017   }
1018
1019   // Compute the opcode for the CMP instruction.
1020   unsigned OpCmp;
1021   LLT Ty = MRI.getType(LhsReg);
1022   switch (Ty.getSizeInBits()) {
1023   default:
1024     return false;
1025   case 32:
1026     OpCmp = X86::UCOMISSrr;
1027     break;
1028   case 64:
1029     OpCmp = X86::UCOMISDrr;
1030     break;
1031   }
1032
1033   Register ResultReg = I.getOperand(0).getReg();
1034   RBI.constrainGenericRegister(
1035       ResultReg,
1036       *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1037   if (SETFOpc) {
1038     MachineInstr &CmpInst =
1039         *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1040              .addReg(LhsReg)
1041              .addReg(RhsReg);
1042
1043     Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1044     Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1045     MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1046                                   TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1047     MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1048                                   TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1049     MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1050                                   TII.get(SETFOpc[2]), ResultReg)
1051                               .addReg(FlagReg1)
1052                               .addReg(FlagReg2);
1053     constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1054     constrainSelectedInstRegOperands(Set1, TII, TRI, RBI);
1055     constrainSelectedInstRegOperands(Set2, TII, TRI, RBI);
1056     constrainSelectedInstRegOperands(Set3, TII, TRI, RBI);
1057
1058     I.eraseFromParent();
1059     return true;
1060   }
1061
1062   X86::CondCode CC;
1063   bool SwapArgs;
1064   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1065   assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1066
1067   if (SwapArgs)
1068     std::swap(LhsReg, RhsReg);
1069
1070   // Emit a compare of LHS/RHS.
1071   MachineInstr &CmpInst =
1072       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1073            .addReg(LhsReg)
1074            .addReg(RhsReg);
1075
1076   MachineInstr &Set =
1077       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1078   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1079   constrainSelectedInstRegOperands(Set, TII, TRI, RBI);
1080   I.eraseFromParent();
1081   return true;
1082 }
1083
1084 bool X86InstructionSelector::selectUadde(MachineInstr &I,
1085                                          MachineRegisterInfo &MRI,
1086                                          MachineFunction &MF) const {
1087   assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1088
1089   const Register DstReg = I.getOperand(0).getReg();
1090   const Register CarryOutReg = I.getOperand(1).getReg();
1091   const Register Op0Reg = I.getOperand(2).getReg();
1092   const Register Op1Reg = I.getOperand(3).getReg();
1093   Register CarryInReg = I.getOperand(4).getReg();
1094
1095   const LLT DstTy = MRI.getType(DstReg);
1096
1097   if (DstTy != LLT::scalar(32))
1098     return false;
1099
1100   // find CarryIn def instruction.
1101   MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1102   while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1103     CarryInReg = Def->getOperand(1).getReg();
1104     Def = MRI.getVRegDef(CarryInReg);
1105   }
1106
1107   unsigned Opcode;
1108   if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1109     // carry set by prev ADD.
1110
1111     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1112         .addReg(CarryInReg);
1113
1114     if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1115       return false;
1116
1117     Opcode = X86::ADC32rr;
1118   } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1119     // carry is constant, support only 0.
1120     if (*val != 0)
1121       return false;
1122
1123     Opcode = X86::ADD32rr;
1124   } else
1125     return false;
1126
1127   MachineInstr &AddInst =
1128       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1129            .addReg(Op0Reg)
1130            .addReg(Op1Reg);
1131
1132   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1133       .addReg(X86::EFLAGS);
1134
1135   if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1136       !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1137     return false;
1138
1139   I.eraseFromParent();
1140   return true;
1141 }
1142
1143 bool X86InstructionSelector::selectExtract(MachineInstr &I,
1144                                            MachineRegisterInfo &MRI,
1145                                            MachineFunction &MF) const {
1146   assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1147          "unexpected instruction");
1148
1149   const Register DstReg = I.getOperand(0).getReg();
1150   const Register SrcReg = I.getOperand(1).getReg();
1151   int64_t Index = I.getOperand(2).getImm();
1152
1153   const LLT DstTy = MRI.getType(DstReg);
1154   const LLT SrcTy = MRI.getType(SrcReg);
1155
1156   // Meanwile handle vector type only.
1157   if (!DstTy.isVector())
1158     return false;
1159
1160   if (Index % DstTy.getSizeInBits() != 0)
1161     return false; // Not extract subvector.
1162
1163   if (Index == 0) {
1164     // Replace by extract subreg copy.
1165     if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1166       return false;
1167
1168     I.eraseFromParent();
1169     return true;
1170   }
1171
1172   bool HasAVX = STI.hasAVX();
1173   bool HasAVX512 = STI.hasAVX512();
1174   bool HasVLX = STI.hasVLX();
1175
1176   if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1177     if (HasVLX)
1178       I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1179     else if (HasAVX)
1180       I.setDesc(TII.get(X86::VEXTRACTF128rr));
1181     else
1182       return false;
1183   } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1184     if (DstTy.getSizeInBits() == 128)
1185       I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1186     else if (DstTy.getSizeInBits() == 256)
1187       I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1188     else
1189       return false;
1190   } else
1191     return false;
1192
1193   // Convert to X86 VEXTRACT immediate.
1194   Index = Index / DstTy.getSizeInBits();
1195   I.getOperand(2).setImm(Index);
1196
1197   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1198 }
1199
1200 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1201                                                MachineInstr &I,
1202                                                MachineRegisterInfo &MRI,
1203                                                MachineFunction &MF) const {
1204   const LLT DstTy = MRI.getType(DstReg);
1205   const LLT SrcTy = MRI.getType(SrcReg);
1206   unsigned SubIdx = X86::NoSubRegister;
1207
1208   if (!DstTy.isVector() || !SrcTy.isVector())
1209     return false;
1210
1211   assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1212          "Incorrect Src/Dst register size");
1213
1214   if (DstTy.getSizeInBits() == 128)
1215     SubIdx = X86::sub_xmm;
1216   else if (DstTy.getSizeInBits() == 256)
1217     SubIdx = X86::sub_ymm;
1218   else
1219     return false;
1220
1221   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1222   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1223
1224   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1225
1226   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1227       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1228     LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
1229     return false;
1230   }
1231
1232   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1233       .addReg(SrcReg, 0, SubIdx);
1234
1235   return true;
1236 }
1237
1238 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1239                                               MachineInstr &I,
1240                                               MachineRegisterInfo &MRI,
1241                                               MachineFunction &MF) const {
1242   const LLT DstTy = MRI.getType(DstReg);
1243   const LLT SrcTy = MRI.getType(SrcReg);
1244   unsigned SubIdx = X86::NoSubRegister;
1245
1246   // TODO: support scalar types
1247   if (!DstTy.isVector() || !SrcTy.isVector())
1248     return false;
1249
1250   assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1251          "Incorrect Src/Dst register size");
1252
1253   if (SrcTy.getSizeInBits() == 128)
1254     SubIdx = X86::sub_xmm;
1255   else if (SrcTy.getSizeInBits() == 256)
1256     SubIdx = X86::sub_ymm;
1257   else
1258     return false;
1259
1260   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1261   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1262
1263   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1264       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1265     LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1266     return false;
1267   }
1268
1269   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1270       .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1271       .addReg(SrcReg);
1272
1273   return true;
1274 }
1275
1276 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1277                                           MachineRegisterInfo &MRI,
1278                                           MachineFunction &MF) const {
1279   assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1280
1281   const Register DstReg = I.getOperand(0).getReg();
1282   const Register SrcReg = I.getOperand(1).getReg();
1283   const Register InsertReg = I.getOperand(2).getReg();
1284   int64_t Index = I.getOperand(3).getImm();
1285
1286   const LLT DstTy = MRI.getType(DstReg);
1287   const LLT InsertRegTy = MRI.getType(InsertReg);
1288
1289   // Meanwile handle vector type only.
1290   if (!DstTy.isVector())
1291     return false;
1292
1293   if (Index % InsertRegTy.getSizeInBits() != 0)
1294     return false; // Not insert subvector.
1295
1296   if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1297     // Replace by subreg copy.
1298     if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1299       return false;
1300
1301     I.eraseFromParent();
1302     return true;
1303   }
1304
1305   bool HasAVX = STI.hasAVX();
1306   bool HasAVX512 = STI.hasAVX512();
1307   bool HasVLX = STI.hasVLX();
1308
1309   if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1310     if (HasVLX)
1311       I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1312     else if (HasAVX)
1313       I.setDesc(TII.get(X86::VINSERTF128rr));
1314     else
1315       return false;
1316   } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1317     if (InsertRegTy.getSizeInBits() == 128)
1318       I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1319     else if (InsertRegTy.getSizeInBits() == 256)
1320       I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1321     else
1322       return false;
1323   } else
1324     return false;
1325
1326   // Convert to X86 VINSERT immediate.
1327   Index = Index / InsertRegTy.getSizeInBits();
1328
1329   I.getOperand(3).setImm(Index);
1330
1331   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1332 }
1333
1334 bool X86InstructionSelector::selectUnmergeValues(
1335     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1336   assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1337          "unexpected instruction");
1338
1339   // Split to extracts.
1340   unsigned NumDefs = I.getNumOperands() - 1;
1341   Register SrcReg = I.getOperand(NumDefs).getReg();
1342   unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1343
1344   for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1345     MachineInstr &ExtrInst =
1346         *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1347                  TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1348              .addReg(SrcReg)
1349              .addImm(Idx * DefSize);
1350
1351     if (!select(ExtrInst))
1352       return false;
1353   }
1354
1355   I.eraseFromParent();
1356   return true;
1357 }
1358
1359 bool X86InstructionSelector::selectMergeValues(
1360     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1361   assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1362           I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1363          "unexpected instruction");
1364
1365   // Split to inserts.
1366   Register DstReg = I.getOperand(0).getReg();
1367   Register SrcReg0 = I.getOperand(1).getReg();
1368
1369   const LLT DstTy = MRI.getType(DstReg);
1370   const LLT SrcTy = MRI.getType(SrcReg0);
1371   unsigned SrcSize = SrcTy.getSizeInBits();
1372
1373   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1374
1375   // For the first src use insertSubReg.
1376   Register DefReg = MRI.createGenericVirtualRegister(DstTy);
1377   MRI.setRegBank(DefReg, RegBank);
1378   if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1379     return false;
1380
1381   for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1382     Register Tmp = MRI.createGenericVirtualRegister(DstTy);
1383     MRI.setRegBank(Tmp, RegBank);
1384
1385     MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1386                                         TII.get(TargetOpcode::G_INSERT), Tmp)
1387                                     .addReg(DefReg)
1388                                     .addReg(I.getOperand(Idx).getReg())
1389                                     .addImm((Idx - 1) * SrcSize);
1390
1391     DefReg = Tmp;
1392
1393     if (!select(InsertInst))
1394       return false;
1395   }
1396
1397   MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1398                                     TII.get(TargetOpcode::COPY), DstReg)
1399                                 .addReg(DefReg);
1400
1401   if (!select(CopyInst))
1402     return false;
1403
1404   I.eraseFromParent();
1405   return true;
1406 }
1407
1408 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1409                                               MachineRegisterInfo &MRI,
1410                                               MachineFunction &MF) const {
1411   assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1412
1413   const Register CondReg = I.getOperand(0).getReg();
1414   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1415
1416   MachineInstr &TestInst =
1417       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1418            .addReg(CondReg)
1419            .addImm(1);
1420   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1421       .addMBB(DestMBB).addImm(X86::COND_NE);
1422
1423   constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1424
1425   I.eraseFromParent();
1426   return true;
1427 }
1428
1429 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1430                                            MachineRegisterInfo &MRI,
1431                                            MachineFunction &MF) const {
1432   assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1433          "unexpected instruction");
1434
1435   // Can't handle alternate code models yet.
1436   CodeModel::Model CM = TM.getCodeModel();
1437   if (CM != CodeModel::Small && CM != CodeModel::Large)
1438     return false;
1439
1440   const Register DstReg = I.getOperand(0).getReg();
1441   const LLT DstTy = MRI.getType(DstReg);
1442   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1443   unsigned Align = DstTy.getSizeInBits();
1444   const DebugLoc &DbgLoc = I.getDebugLoc();
1445
1446   unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
1447
1448   // Create the load from the constant pool.
1449   const ConstantFP *CFP = I.getOperand(1).getFPImm();
1450   unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
1451   MachineInstr *LoadInst = nullptr;
1452   unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1453
1454   if (CM == CodeModel::Large && STI.is64Bit()) {
1455     // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1456     // they cannot be folded into immediate fields.
1457
1458     Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1459     BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1460         .addConstantPoolIndex(CPI, 0, OpFlag);
1461
1462     MachineMemOperand *MMO = MF.getMachineMemOperand(
1463         MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1464         MF.getDataLayout().getPointerSize(), Align);
1465
1466     LoadInst =
1467         addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1468                      AddrReg)
1469             .addMemOperand(MMO);
1470
1471   } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1472     // Handle the case when globals fit in our immediate field.
1473     // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1474
1475     // x86-32 PIC requires a PIC base register for constant pools.
1476     unsigned PICBase = 0;
1477     if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1478       // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1479       // In DAGISEL the code that initialize it generated by the CGBR pass.
1480       return false; // TODO support the mode.
1481     } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1482       PICBase = X86::RIP;
1483
1484     LoadInst = addConstantPoolReference(
1485         BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1486         OpFlag);
1487   } else
1488     return false;
1489
1490   constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1491   I.eraseFromParent();
1492   return true;
1493 }
1494
1495 bool X86InstructionSelector::selectImplicitDefOrPHI(
1496     MachineInstr &I, MachineRegisterInfo &MRI) const {
1497   assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1498           I.getOpcode() == TargetOpcode::G_PHI) &&
1499          "unexpected instruction");
1500
1501   Register DstReg = I.getOperand(0).getReg();
1502
1503   if (!MRI.getRegClassOrNull(DstReg)) {
1504     const LLT DstTy = MRI.getType(DstReg);
1505     const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1506
1507     if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1508       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1509                         << " operand\n");
1510       return false;
1511     }
1512   }
1513
1514   if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1515     I.setDesc(TII.get(X86::IMPLICIT_DEF));
1516   else
1517     I.setDesc(TII.get(X86::PHI));
1518
1519   return true;
1520 }
1521
1522 // Currently GlobalIsel TableGen generates patterns for shift imm and shift 1,
1523 // but with shiftCount i8. In G_LSHR/G_ASHR/G_SHL like LLVM-IR both arguments
1524 // has the same type, so for now only shift i8 can use auto generated
1525 // TableGen patterns.
1526 bool X86InstructionSelector::selectShift(MachineInstr &I,
1527                                          MachineRegisterInfo &MRI,
1528                                          MachineFunction &MF) const {
1529
1530   assert((I.getOpcode() == TargetOpcode::G_SHL ||
1531           I.getOpcode() == TargetOpcode::G_ASHR ||
1532           I.getOpcode() == TargetOpcode::G_LSHR) &&
1533          "unexpected instruction");
1534
1535   Register DstReg = I.getOperand(0).getReg();
1536   const LLT DstTy = MRI.getType(DstReg);
1537   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1538
1539   const static struct ShiftEntry {
1540     unsigned SizeInBits;
1541     unsigned OpLSHR;
1542     unsigned OpASHR;
1543     unsigned OpSHL;
1544   } OpTable[] = {
1545       {8, X86::SHR8rCL, X86::SAR8rCL, X86::SHL8rCL},     // i8
1546       {16, X86::SHR16rCL, X86::SAR16rCL, X86::SHL16rCL}, // i16
1547       {32, X86::SHR32rCL, X86::SAR32rCL, X86::SHL32rCL}, // i32
1548       {64, X86::SHR64rCL, X86::SAR64rCL, X86::SHL64rCL}  // i64
1549   };
1550
1551   if (DstRB.getID() != X86::GPRRegBankID)
1552     return false;
1553
1554   auto ShiftEntryIt = std::find_if(
1555       std::begin(OpTable), std::end(OpTable), [DstTy](const ShiftEntry &El) {
1556         return El.SizeInBits == DstTy.getSizeInBits();
1557       });
1558   if (ShiftEntryIt == std::end(OpTable))
1559     return false;
1560
1561   unsigned Opcode = 0;
1562   switch (I.getOpcode()) {
1563   case TargetOpcode::G_SHL:
1564     Opcode = ShiftEntryIt->OpSHL;
1565     break;
1566   case TargetOpcode::G_ASHR:
1567     Opcode = ShiftEntryIt->OpASHR;
1568     break;
1569   case TargetOpcode::G_LSHR:
1570     Opcode = ShiftEntryIt->OpLSHR;
1571     break;
1572   default:
1573     return false;
1574   }
1575
1576   Register Op0Reg = I.getOperand(1).getReg();
1577   Register Op1Reg = I.getOperand(2).getReg();
1578
1579   assert(MRI.getType(Op1Reg).getSizeInBits() == 8);
1580
1581   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1582           X86::CL)
1583     .addReg(Op1Reg);
1584
1585   MachineInstr &ShiftInst =
1586       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1587            .addReg(Op0Reg);
1588
1589   constrainSelectedInstRegOperands(ShiftInst, TII, TRI, RBI);
1590   I.eraseFromParent();
1591   return true;
1592 }
1593
1594 bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1595                                           MachineRegisterInfo &MRI,
1596                                           MachineFunction &MF) const {
1597   // The implementation of this function is taken from X86FastISel.
1598   assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1599           I.getOpcode() == TargetOpcode::G_SREM ||
1600           I.getOpcode() == TargetOpcode::G_UDIV ||
1601           I.getOpcode() == TargetOpcode::G_UREM) &&
1602          "unexpected instruction");
1603
1604   const Register DstReg = I.getOperand(0).getReg();
1605   const Register Op1Reg = I.getOperand(1).getReg();
1606   const Register Op2Reg = I.getOperand(2).getReg();
1607
1608   const LLT RegTy = MRI.getType(DstReg);
1609   assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1610          "Arguments and return value types must match");
1611
1612   const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1613   if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1614     return false;
1615
1616   const static unsigned NumTypes = 4; // i8, i16, i32, i64
1617   const static unsigned NumOps = 4;   // SDiv, SRem, UDiv, URem
1618   const static bool S = true;         // IsSigned
1619   const static bool U = false;        // !IsSigned
1620   const static unsigned Copy = TargetOpcode::COPY;
1621   // For the X86 IDIV instruction, in most cases the dividend
1622   // (numerator) must be in a specific register pair highreg:lowreg,
1623   // producing the quotient in lowreg and the remainder in highreg.
1624   // For most data types, to set up the instruction, the dividend is
1625   // copied into lowreg, and lowreg is sign-extended into highreg.  The
1626   // exception is i8, where the dividend is defined as a single register rather
1627   // than a register pair, and we therefore directly sign-extend the dividend
1628   // into lowreg, instead of copying, and ignore the highreg.
1629   const static struct DivRemEntry {
1630     // The following portion depends only on the data type.
1631     unsigned SizeInBits;
1632     unsigned LowInReg;  // low part of the register pair
1633     unsigned HighInReg; // high part of the register pair
1634     // The following portion depends on both the data type and the operation.
1635     struct DivRemResult {
1636       unsigned OpDivRem;        // The specific DIV/IDIV opcode to use.
1637       unsigned OpSignExtend;    // Opcode for sign-extending lowreg into
1638                                 // highreg, or copying a zero into highreg.
1639       unsigned OpCopy;          // Opcode for copying dividend into lowreg, or
1640                                 // zero/sign-extending into lowreg for i8.
1641       unsigned DivRemResultReg; // Register containing the desired result.
1642       bool IsOpSigned;          // Whether to use signed or unsigned form.
1643     } ResultTable[NumOps];
1644   } OpTable[NumTypes] = {
1645       {8,
1646        X86::AX,
1647        0,
1648        {
1649            {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1650            {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1651            {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U},  // UDiv
1652            {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U},  // URem
1653        }},                                                // i8
1654       {16,
1655        X86::AX,
1656        X86::DX,
1657        {
1658            {X86::IDIV16r, X86::CWD, Copy, X86::AX, S},    // SDiv
1659            {X86::IDIV16r, X86::CWD, Copy, X86::DX, S},    // SRem
1660            {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1661            {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1662        }},                                                // i16
1663       {32,
1664        X86::EAX,
1665        X86::EDX,
1666        {
1667            {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S},    // SDiv
1668            {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S},    // SRem
1669            {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1670            {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1671        }},                                                 // i32
1672       {64,
1673        X86::RAX,
1674        X86::RDX,
1675        {
1676            {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S},    // SDiv
1677            {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S},    // SRem
1678            {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1679            {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1680        }},                                                 // i64
1681   };
1682
1683   auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1684                                 [RegTy](const DivRemEntry &El) {
1685                                   return El.SizeInBits == RegTy.getSizeInBits();
1686                                 });
1687   if (OpEntryIt == std::end(OpTable))
1688     return false;
1689
1690   unsigned OpIndex;
1691   switch (I.getOpcode()) {
1692   default:
1693     llvm_unreachable("Unexpected div/rem opcode");
1694   case TargetOpcode::G_SDIV:
1695     OpIndex = 0;
1696     break;
1697   case TargetOpcode::G_SREM:
1698     OpIndex = 1;
1699     break;
1700   case TargetOpcode::G_UDIV:
1701     OpIndex = 2;
1702     break;
1703   case TargetOpcode::G_UREM:
1704     OpIndex = 3;
1705     break;
1706   }
1707
1708   const DivRemEntry &TypeEntry = *OpEntryIt;
1709   const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1710
1711   const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1712   if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1713       !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1714       !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1715     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1716                       << " operand\n");
1717     return false;
1718   }
1719
1720   // Move op1 into low-order input register.
1721   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1722           TypeEntry.LowInReg)
1723       .addReg(Op1Reg);
1724   // Zero-extend or sign-extend into high-order input register.
1725   if (OpEntry.OpSignExtend) {
1726     if (OpEntry.IsOpSigned)
1727       BuildMI(*I.getParent(), I, I.getDebugLoc(),
1728               TII.get(OpEntry.OpSignExtend));
1729     else {
1730       Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1731       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1732               Zero32);
1733
1734       // Copy the zero into the appropriate sub/super/identical physical
1735       // register. Unfortunately the operations needed are not uniform enough
1736       // to fit neatly into the table above.
1737       if (RegTy.getSizeInBits() == 16) {
1738         BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1739                 TypeEntry.HighInReg)
1740             .addReg(Zero32, 0, X86::sub_16bit);
1741       } else if (RegTy.getSizeInBits() == 32) {
1742         BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1743                 TypeEntry.HighInReg)
1744             .addReg(Zero32);
1745       } else if (RegTy.getSizeInBits() == 64) {
1746         BuildMI(*I.getParent(), I, I.getDebugLoc(),
1747                 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1748             .addImm(0)
1749             .addReg(Zero32)
1750             .addImm(X86::sub_32bit);
1751       }
1752     }
1753   }
1754   // Generate the DIV/IDIV instruction.
1755   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1756       .addReg(Op2Reg);
1757   // For i8 remainder, we can't reference ah directly, as we'll end
1758   // up with bogus copies like %r9b = COPY %ah. Reference ax
1759   // instead to prevent ah references in a rex instruction.
1760   //
1761   // The current assumption of the fast register allocator is that isel
1762   // won't generate explicit references to the GR8_NOREX registers. If
1763   // the allocator and/or the backend get enhanced to be more robust in
1764   // that regard, this can be, and should be, removed.
1765   if ((I.getOpcode() == Instruction::SRem ||
1766        I.getOpcode() == Instruction::URem) &&
1767       OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) {
1768     Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1769     Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1770     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1771         .addReg(X86::AX);
1772
1773     // Shift AX right by 8 bits instead of using AH.
1774     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1775             ResultSuperReg)
1776         .addReg(SourceSuperReg)
1777         .addImm(8);
1778
1779     // Now reference the 8-bit subreg of the result.
1780     BuildMI(*I.getParent(), I, I.getDebugLoc(),
1781             TII.get(TargetOpcode::SUBREG_TO_REG))
1782         .addDef(DstReg)
1783         .addImm(0)
1784         .addReg(ResultSuperReg)
1785         .addImm(X86::sub_8bit);
1786   } else {
1787     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1788             DstReg)
1789         .addReg(OpEntry.DivRemResultReg);
1790   }
1791   I.eraseFromParent();
1792   return true;
1793 }
1794
1795 bool X86InstructionSelector::selectIntrinsicWSideEffects(
1796     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1797
1798   assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1799          "unexpected instruction");
1800
1801   if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1802     return false;
1803
1804   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1805
1806   I.eraseFromParent();
1807   return true;
1808 }
1809
1810 InstructionSelector *
1811 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1812                                    X86Subtarget &Subtarget,
1813                                    X86RegisterBankInfo &RBI) {
1814   return new X86InstructionSelector(TM, Subtarget, RBI);
1815 }