lib/Target/AMDGPU/SIInstrInfo.h

   1 //===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// Interface definition for SIInstrInfo.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
  15 #define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
  16
  17 #include "AMDGPUInstrInfo.h"
  18 #include "SIDefines.h"
  19 #include "SIRegisterInfo.h"
  20 #include "Utils/AMDGPUBaseInfo.h"
  21 #include "llvm/ADT/ArrayRef.h"
  22 #include "llvm/ADT/SetVector.h"
  23 #include "llvm/CodeGen/MachineBasicBlock.h"
  24 #include "llvm/CodeGen/MachineFunction.h"
  25 #include "llvm/CodeGen/MachineInstr.h"
  26 #include "llvm/CodeGen/MachineInstrBuilder.h"
  27 #include "llvm/CodeGen/MachineOperand.h"
  28 #include "llvm/MC/MCInstrDesc.h"
  29 #include "llvm/Support/Compiler.h"
  30 #include <cassert>
  31 #include <cstdint>
  32
  33 #define GET_INSTRINFO_HEADER
  34 #include "AMDGPUGenInstrInfo.inc"
  35
  36 namespace llvm {
  37
  38 class APInt;
  39 class MachineDominatorTree;
  40 class MachineRegisterInfo;
  41 class RegScavenger;
  42 class GCNSubtarget;
  43 class TargetRegisterClass;
  44
  45 class SIInstrInfo final : public AMDGPUGenInstrInfo {
  46 private:
  47   const SIRegisterInfo RI;
  48   const GCNSubtarget &ST;
  49
  50   // The inverse predicate should have the negative value.
  51   enum BranchPredicate {
  52     INVALID_BR = 0,
  53     SCC_TRUE = 1,
  54     SCC_FALSE = -1,
  55     VCCNZ = 2,
  56     VCCZ = -2,
  57     EXECNZ = -3,
  58     EXECZ = 3
  59   };
  60
  61   using SetVectorType = SmallSetVector<MachineInstr *, 32>;
  62
  63   static unsigned getBranchOpcode(BranchPredicate Cond);
  64   static BranchPredicate getBranchPredicate(unsigned Opcode);
  65
  66 public:
  67   unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
  68                               MachineRegisterInfo &MRI,
  69                               MachineOperand &SuperReg,
  70                               const TargetRegisterClass *SuperRC,
  71                               unsigned SubIdx,
  72                               const TargetRegisterClass *SubRC) const;
  73   MachineOperand buildExtractSubRegOrImm(MachineBasicBlock::iterator MI,
  74                                          MachineRegisterInfo &MRI,
  75                                          MachineOperand &SuperReg,
  76                                          const TargetRegisterClass *SuperRC,
  77                                          unsigned SubIdx,
  78                                          const TargetRegisterClass *SubRC) const;
  79 private:
  80   void swapOperands(MachineInstr &Inst) const;
  81
  82   bool moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
  83                         MachineDominatorTree *MDT = nullptr) const;
  84
  85   void lowerScalarAbs(SetVectorType &Worklist,
  86                       MachineInstr &Inst) const;
  87
  88   void lowerScalarXnor(SetVectorType &Worklist,
  89                        MachineInstr &Inst) const;
  90
  91   void splitScalarNotBinop(SetVectorType &Worklist,
  92                            MachineInstr &Inst,
  93                            unsigned Opcode) const;
  94
  95   void splitScalarBinOpN2(SetVectorType &Worklist,
  96                           MachineInstr &Inst,
  97                           unsigned Opcode) const;
  98
  99   void splitScalar64BitUnaryOp(SetVectorType &Worklist,
 100                                MachineInstr &Inst, unsigned Opcode) const;
 101
 102   void splitScalar64BitAddSub(SetVectorType &Worklist, MachineInstr &Inst,
 103                               MachineDominatorTree *MDT = nullptr) const;
 104
 105   void splitScalar64BitBinaryOp(SetVectorType &Worklist, MachineInstr &Inst,
 106                                 unsigned Opcode,
 107                                 MachineDominatorTree *MDT = nullptr) const;
 108
 109   void splitScalar64BitXnor(SetVectorType &Worklist, MachineInstr &Inst,
 110                                 MachineDominatorTree *MDT = nullptr) const;
 111
 112   void splitScalar64BitBCNT(SetVectorType &Worklist,
 113                             MachineInstr &Inst) const;
 114   void splitScalar64BitBFE(SetVectorType &Worklist,
 115                            MachineInstr &Inst) const;
 116   void movePackToVALU(SetVectorType &Worklist,
 117                       MachineRegisterInfo &MRI,
 118                       MachineInstr &Inst) const;
 119
 120   void addUsersToMoveToVALUWorklist(unsigned Reg, MachineRegisterInfo &MRI,
 121                                     SetVectorType &Worklist) const;
 122
 123   void
 124   addSCCDefUsersToVALUWorklist(MachineInstr &SCCDefInst,
 125                                SetVectorType &Worklist) const;
 126
 127   const TargetRegisterClass *
 128   getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
 129
 130   bool checkInstOffsetsDoNotOverlap(MachineInstr &MIa, MachineInstr &MIb) const;
 131
 132   unsigned findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
 133
 134 protected:
 135   bool swapSourceModifiers(MachineInstr &MI,
 136                            MachineOperand &Src0, unsigned Src0OpName,
 137                            MachineOperand &Src1, unsigned Src1OpName) const;
 138
 139   MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
 140                                        unsigned OpIdx0,
 141                                        unsigned OpIdx1) const override;
 142
 143 public:
 144   enum TargetOperandFlags {
 145     MO_MASK = 0x7,
 146
 147     MO_NONE = 0,
 148     // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
 149     MO_GOTPCREL = 1,
 150     // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
 151     MO_GOTPCREL32 = 2,
 152     MO_GOTPCREL32_LO = 2,
 153     // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
 154     MO_GOTPCREL32_HI = 3,
 155     // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
 156     MO_REL32 = 4,
 157     MO_REL32_LO = 4,
 158     // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
 159     MO_REL32_HI = 5
 160   };
 161
 162   explicit SIInstrInfo(const GCNSubtarget &ST);
 163
 164   const SIRegisterInfo &getRegisterInfo() const {
 165     return RI;
 166   }
 167
 168   bool isReallyTriviallyReMaterializable(const MachineInstr &MI,
 169                                          AliasAnalysis *AA) const override;
 170
 171   bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
 172                                int64_t &Offset1,
 173                                int64_t &Offset2) const override;
 174
 175   bool getMemOperandWithOffset(MachineInstr &LdSt, MachineOperand *&BaseOp,
 176                                int64_t &Offset,
 177                                const TargetRegisterInfo *TRI) const final;
 178
 179   bool shouldClusterMemOps(MachineOperand &BaseOp1, MachineOperand &BaseOp2,
 180                            unsigned NumLoads) const override;
 181
 182   bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
 183                                int64_t Offset1, unsigned NumLoads) const override;
 184
 185   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
 186                    const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
 187                    bool KillSrc) const override;
 188
 189   unsigned calculateLDSSpillAddress(MachineBasicBlock &MBB, MachineInstr &MI,
 190                                     RegScavenger *RS, unsigned TmpReg,
 191                                     unsigned Offset, unsigned Size) const;
 192
 193   void materializeImmediate(MachineBasicBlock &MBB,
 194                             MachineBasicBlock::iterator MI,
 195                             const DebugLoc &DL,
 196                             unsigned DestReg,
 197                             int64_t Value) const;
 198
 199   const TargetRegisterClass *getPreferredSelectRegClass(
 200                                unsigned Size) const;
 201
 202   unsigned insertNE(MachineBasicBlock *MBB,
 203                     MachineBasicBlock::iterator I, const DebugLoc &DL,
 204                     unsigned SrcReg, int Value) const;
 205
 206   unsigned insertEQ(MachineBasicBlock *MBB,
 207                     MachineBasicBlock::iterator I, const DebugLoc &DL,
 208                     unsigned SrcReg, int Value)  const;
 209
 210   void storeRegToStackSlot(MachineBasicBlock &MBB,
 211                            MachineBasicBlock::iterator MI, unsigned SrcReg,
 212                            bool isKill, int FrameIndex,
 213                            const TargetRegisterClass *RC,
 214                            const TargetRegisterInfo *TRI) const override;
 215
 216   void loadRegFromStackSlot(MachineBasicBlock &MBB,
 217                             MachineBasicBlock::iterator MI, unsigned DestReg,
 218                             int FrameIndex, const TargetRegisterClass *RC,
 219                             const TargetRegisterInfo *TRI) const override;
 220
 221   bool expandPostRAPseudo(MachineInstr &MI) const override;
 222
 223   // Returns an opcode that can be used to move a value to a \p DstRC
 224   // register.  If there is no hardware instruction that can store to \p
 225   // DstRC, then AMDGPU::COPY is returned.
 226   unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
 227
 228   LLVM_READONLY
 229   int commuteOpcode(unsigned Opc) const;
 230
 231   LLVM_READONLY
 232   inline int commuteOpcode(const MachineInstr &MI) const {
 233     return commuteOpcode(MI.getOpcode());
 234   }
 235
 236   bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
 237                              unsigned &SrcOpIdx2) const override;
 238
 239   bool findCommutedOpIndices(MCInstrDesc Desc, unsigned & SrcOpIdx0,
 240    unsigned & SrcOpIdx1) const;
 241
 242   bool isBranchOffsetInRange(unsigned BranchOpc,
 243                              int64_t BrOffset) const override;
 244
 245   MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
 246
 247   unsigned insertIndirectBranch(MachineBasicBlock &MBB,
 248                                 MachineBasicBlock &NewDestBB,
 249                                 const DebugLoc &DL,
 250                                 int64_t BrOffset,
 251                                 RegScavenger *RS = nullptr) const override;
 252
 253   bool analyzeBranchImpl(MachineBasicBlock &MBB,
 254                          MachineBasicBlock::iterator I,
 255                          MachineBasicBlock *&TBB,
 256                          MachineBasicBlock *&FBB,
 257                          SmallVectorImpl<MachineOperand> &Cond,
 258                          bool AllowModify) const;
 259
 260   bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
 261                      MachineBasicBlock *&FBB,
 262                      SmallVectorImpl<MachineOperand> &Cond,
 263                      bool AllowModify = false) const override;
 264
 265   unsigned removeBranch(MachineBasicBlock &MBB,
 266                         int *BytesRemoved = nullptr) const override;
 267
 268   unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 269                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
 270                         const DebugLoc &DL,
 271                         int *BytesAdded = nullptr) const override;
 272
 273   bool reverseBranchCondition(
 274     SmallVectorImpl<MachineOperand> &Cond) const override;
 275
 276   bool canInsertSelect(const MachineBasicBlock &MBB,
 277                        ArrayRef<MachineOperand> Cond,
 278                        unsigned TrueReg, unsigned FalseReg,
 279                        int &CondCycles,
 280                        int &TrueCycles, int &FalseCycles) const override;
 281
 282   void insertSelect(MachineBasicBlock &MBB,
 283                     MachineBasicBlock::iterator I, const DebugLoc &DL,
 284                     unsigned DstReg, ArrayRef<MachineOperand> Cond,
 285                     unsigned TrueReg, unsigned FalseReg) const override;
 286
 287   void insertVectorSelect(MachineBasicBlock &MBB,
 288                           MachineBasicBlock::iterator I, const DebugLoc &DL,
 289                           unsigned DstReg, ArrayRef<MachineOperand> Cond,
 290                           unsigned TrueReg, unsigned FalseReg) const;
 291
 292   unsigned getAddressSpaceForPseudoSourceKind(
 293              unsigned Kind) const override;
 294
 295   bool
 296   areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb,
 297                                   AliasAnalysis *AA = nullptr) const override;
 298
 299   bool isFoldableCopy(const MachineInstr &MI) const;
 300
 301   bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, unsigned Reg,
 302                      MachineRegisterInfo *MRI) const final;
 303
 304   unsigned getMachineCSELookAheadLimit() const override { return 500; }
 305
 306   MachineInstr *convertToThreeAddress(MachineFunction::iterator &MBB,
 307                                       MachineInstr &MI,
 308                                       LiveVariables *LV) const override;
 309
 310   bool isSchedulingBoundary(const MachineInstr &MI,
 311                             const MachineBasicBlock *MBB,
 312                             const MachineFunction &MF) const override;
 313
 314   static bool isSALU(const MachineInstr &MI) {
 315     return MI.getDesc().TSFlags & SIInstrFlags::SALU;
 316   }
 317
 318   bool isSALU(uint16_t Opcode) const {
 319     return get(Opcode).TSFlags & SIInstrFlags::SALU;
 320   }
 321
 322   static bool isVALU(const MachineInstr &MI) {
 323     return MI.getDesc().TSFlags & SIInstrFlags::VALU;
 324   }
 325
 326   bool isVALU(uint16_t Opcode) const {
 327     return get(Opcode).TSFlags & SIInstrFlags::VALU;
 328   }
 329
 330   static bool isVMEM(const MachineInstr &MI) {
 331     return isMUBUF(MI) || isMTBUF(MI) || isMIMG(MI);
 332   }
 333
 334   bool isVMEM(uint16_t Opcode) const {
 335     return isMUBUF(Opcode) || isMTBUF(Opcode) || isMIMG(Opcode);
 336   }
 337
 338   static bool isSOP1(const MachineInstr &MI) {
 339     return MI.getDesc().TSFlags & SIInstrFlags::SOP1;
 340   }
 341
 342   bool isSOP1(uint16_t Opcode) const {
 343     return get(Opcode).TSFlags & SIInstrFlags::SOP1;
 344   }
 345
 346   static bool isSOP2(const MachineInstr &MI) {
 347     return MI.getDesc().TSFlags & SIInstrFlags::SOP2;
 348   }
 349
 350   bool isSOP2(uint16_t Opcode) const {
 351     return get(Opcode).TSFlags & SIInstrFlags::SOP2;
 352   }
 353
 354   static bool isSOPC(const MachineInstr &MI) {
 355     return MI.getDesc().TSFlags & SIInstrFlags::SOPC;
 356   }
 357
 358   bool isSOPC(uint16_t Opcode) const {
 359     return get(Opcode).TSFlags & SIInstrFlags::SOPC;
 360   }
 361
 362   static bool isSOPK(const MachineInstr &MI) {
 363     return MI.getDesc().TSFlags & SIInstrFlags::SOPK;
 364   }
 365
 366   bool isSOPK(uint16_t Opcode) const {
 367     return get(Opcode).TSFlags & SIInstrFlags::SOPK;
 368   }
 369
 370   static bool isSOPP(const MachineInstr &MI) {
 371     return MI.getDesc().TSFlags & SIInstrFlags::SOPP;
 372   }
 373
 374   bool isSOPP(uint16_t Opcode) const {
 375     return get(Opcode).TSFlags & SIInstrFlags::SOPP;
 376   }
 377
 378   static bool isVOP1(const MachineInstr &MI) {
 379     return MI.getDesc().TSFlags & SIInstrFlags::VOP1;
 380   }
 381
 382   bool isVOP1(uint16_t Opcode) const {
 383     return get(Opcode).TSFlags & SIInstrFlags::VOP1;
 384   }
 385
 386   static bool isVOP2(const MachineInstr &MI) {
 387     return MI.getDesc().TSFlags & SIInstrFlags::VOP2;
 388   }
 389
 390   bool isVOP2(uint16_t Opcode) const {
 391     return get(Opcode).TSFlags & SIInstrFlags::VOP2;
 392   }
 393
 394   static bool isVOP3(const MachineInstr &MI) {
 395     return MI.getDesc().TSFlags & SIInstrFlags::VOP3;
 396   }
 397
 398   bool isVOP3(uint16_t Opcode) const {
 399     return get(Opcode).TSFlags & SIInstrFlags::VOP3;
 400   }
 401
 402   static bool isSDWA(const MachineInstr &MI) {
 403     return MI.getDesc().TSFlags & SIInstrFlags::SDWA;
 404   }
 405
 406   bool isSDWA(uint16_t Opcode) const {
 407     return get(Opcode).TSFlags & SIInstrFlags::SDWA;
 408   }
 409
 410   static bool isVOPC(const MachineInstr &MI) {
 411     return MI.getDesc().TSFlags & SIInstrFlags::VOPC;
 412   }
 413
 414   bool isVOPC(uint16_t Opcode) const {
 415     return get(Opcode).TSFlags & SIInstrFlags::VOPC;
 416   }
 417
 418   static bool isMUBUF(const MachineInstr &MI) {
 419     return MI.getDesc().TSFlags & SIInstrFlags::MUBUF;
 420   }
 421
 422   bool isMUBUF(uint16_t Opcode) const {
 423     return get(Opcode).TSFlags & SIInstrFlags::MUBUF;
 424   }
 425
 426   static bool isMTBUF(const MachineInstr &MI) {
 427     return MI.getDesc().TSFlags & SIInstrFlags::MTBUF;
 428   }
 429
 430   bool isMTBUF(uint16_t Opcode) const {
 431     return get(Opcode).TSFlags & SIInstrFlags::MTBUF;
 432   }
 433
 434   static bool isSMRD(const MachineInstr &MI) {
 435     return MI.getDesc().TSFlags & SIInstrFlags::SMRD;
 436   }
 437
 438   bool isSMRD(uint16_t Opcode) const {
 439     return get(Opcode).TSFlags & SIInstrFlags::SMRD;
 440   }
 441
 442   bool isBufferSMRD(const MachineInstr &MI) const;
 443
 444   static bool isDS(const MachineInstr &MI) {
 445     return MI.getDesc().TSFlags & SIInstrFlags::DS;
 446   }
 447
 448   bool isDS(uint16_t Opcode) const {
 449     return get(Opcode).TSFlags & SIInstrFlags::DS;
 450   }
 451
 452   bool isAlwaysGDS(uint16_t Opcode) const;
 453
 454   static bool isMIMG(const MachineInstr &MI) {
 455     return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
 456   }
 457
 458   bool isMIMG(uint16_t Opcode) const {
 459     return get(Opcode).TSFlags & SIInstrFlags::MIMG;
 460   }
 461
 462   static bool isGather4(const MachineInstr &MI) {
 463     return MI.getDesc().TSFlags & SIInstrFlags::Gather4;
 464   }
 465
 466   bool isGather4(uint16_t Opcode) const {
 467     return get(Opcode).TSFlags & SIInstrFlags::Gather4;
 468   }
 469
 470   static bool isFLAT(const MachineInstr &MI) {
 471     return MI.getDesc().TSFlags & SIInstrFlags::FLAT;
 472   }
 473
 474   // Is a FLAT encoded instruction which accesses a specific segment,
 475   // i.e. global_* or scratch_*.
 476   static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
 477     auto Flags = MI.getDesc().TSFlags;
 478     return (Flags & SIInstrFlags::FLAT) && !(Flags & SIInstrFlags::LGKM_CNT);
 479   }
 480
 481   // Any FLAT encoded instruction, including global_* and scratch_*.
 482   bool isFLAT(uint16_t Opcode) const {
 483     return get(Opcode).TSFlags & SIInstrFlags::FLAT;
 484   }
 485
 486   static bool isEXP(const MachineInstr &MI) {
 487     return MI.getDesc().TSFlags & SIInstrFlags::EXP;
 488   }
 489
 490   bool isEXP(uint16_t Opcode) const {
 491     return get(Opcode).TSFlags & SIInstrFlags::EXP;
 492   }
 493
 494   static bool isWQM(const MachineInstr &MI) {
 495     return MI.getDesc().TSFlags & SIInstrFlags::WQM;
 496   }
 497
 498   bool isWQM(uint16_t Opcode) const {
 499     return get(Opcode).TSFlags & SIInstrFlags::WQM;
 500   }
 501
 502   static bool isDisableWQM(const MachineInstr &MI) {
 503     return MI.getDesc().TSFlags & SIInstrFlags::DisableWQM;
 504   }
 505
 506   bool isDisableWQM(uint16_t Opcode) const {
 507     return get(Opcode).TSFlags & SIInstrFlags::DisableWQM;
 508   }
 509
 510   static bool isVGPRSpill(const MachineInstr &MI) {
 511     return MI.getDesc().TSFlags & SIInstrFlags::VGPRSpill;
 512   }
 513
 514   bool isVGPRSpill(uint16_t Opcode) const {
 515     return get(Opcode).TSFlags & SIInstrFlags::VGPRSpill;
 516   }
 517
 518   static bool isSGPRSpill(const MachineInstr &MI) {
 519     return MI.getDesc().TSFlags & SIInstrFlags::SGPRSpill;
 520   }
 521
 522   bool isSGPRSpill(uint16_t Opcode) const {
 523     return get(Opcode).TSFlags & SIInstrFlags::SGPRSpill;
 524   }
 525
 526   static bool isDPP(const MachineInstr &MI) {
 527     return MI.getDesc().TSFlags & SIInstrFlags::DPP;
 528   }
 529
 530   bool isDPP(uint16_t Opcode) const {
 531     return get(Opcode).TSFlags & SIInstrFlags::DPP;
 532   }
 533
 534   static bool isVOP3P(const MachineInstr &MI) {
 535     return MI.getDesc().TSFlags & SIInstrFlags::VOP3P;
 536   }
 537
 538   bool isVOP3P(uint16_t Opcode) const {
 539     return get(Opcode).TSFlags & SIInstrFlags::VOP3P;
 540   }
 541
 542   static bool isVINTRP(const MachineInstr &MI) {
 543     return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;
 544   }
 545
 546   bool isVINTRP(uint16_t Opcode) const {
 547     return get(Opcode).TSFlags & SIInstrFlags::VINTRP;
 548   }
 549
 550   static bool isScalarUnit(const MachineInstr &MI) {
 551     return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
 552   }
 553
 554   static bool usesVM_CNT(const MachineInstr &MI) {
 555     return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;
 556   }
 557
 558   static bool usesLGKM_CNT(const MachineInstr &MI) {
 559     return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;
 560   }
 561
 562   static bool sopkIsZext(const MachineInstr &MI) {
 563     return MI.getDesc().TSFlags & SIInstrFlags::SOPK_ZEXT;
 564   }
 565
 566   bool sopkIsZext(uint16_t Opcode) const {
 567     return get(Opcode).TSFlags & SIInstrFlags::SOPK_ZEXT;
 568   }
 569
 570   /// \returns true if this is an s_store_dword* instruction. This is more
 571   /// specific than than isSMEM && mayStore.
 572   static bool isScalarStore(const MachineInstr &MI) {
 573     return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;
 574   }
 575
 576   bool isScalarStore(uint16_t Opcode) const {
 577     return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;
 578   }
 579
 580   static bool isFixedSize(const MachineInstr &MI) {
 581     return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;
 582   }
 583
 584   bool isFixedSize(uint16_t Opcode) const {
 585     return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;
 586   }
 587
 588   static bool hasFPClamp(const MachineInstr &MI) {
 589     return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
 590   }
 591
 592   bool hasFPClamp(uint16_t Opcode) const {
 593     return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
 594   }
 595
 596   static bool hasIntClamp(const MachineInstr &MI) {
 597     return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
 598   }
 599
 600   uint64_t getClampMask(const MachineInstr &MI) const {
 601     const uint64_t ClampFlags = SIInstrFlags::FPClamp |
 602                                 SIInstrFlags::IntClamp |
 603                                 SIInstrFlags::ClampLo |
 604                                 SIInstrFlags::ClampHi;
 605       return MI.getDesc().TSFlags & ClampFlags;
 606   }
 607
 608   static bool usesFPDPRounding(const MachineInstr &MI) {
 609     return MI.getDesc().TSFlags & SIInstrFlags::FPDPRounding;
 610   }
 611
 612   bool usesFPDPRounding(uint16_t Opcode) const {
 613     return get(Opcode).TSFlags & SIInstrFlags::FPDPRounding;
 614   }
 615
 616   bool isVGPRCopy(const MachineInstr &MI) const {
 617     assert(MI.isCopy());
 618     unsigned Dest = MI.getOperand(0).getReg();
 619     const MachineFunction &MF = *MI.getParent()->getParent();
 620     const MachineRegisterInfo &MRI = MF.getRegInfo();
 621     return !RI.isSGPRReg(MRI, Dest);
 622   }
 623
 624   /// Whether we must prevent this instruction from executing with EXEC = 0.
 625   bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
 626
 627   bool isInlineConstant(const APInt &Imm) const;
 628
 629   bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const;
 630
 631   bool isInlineConstant(const MachineOperand &MO,
 632                         const MCOperandInfo &OpInfo) const {
 633     return isInlineConstant(MO, OpInfo.OperandType);
 634   }
 635
 636   /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
 637   /// be an inline immediate.
 638   bool isInlineConstant(const MachineInstr &MI,
 639                         const MachineOperand &UseMO,
 640                         const MachineOperand &DefMO) const {
 641     assert(UseMO.getParent() == &MI);
 642     int OpIdx = MI.getOperandNo(&UseMO);
 643     if (!MI.getDesc().OpInfo || OpIdx >= MI.getDesc().NumOperands) {
 644       return false;
 645     }
 646
 647     return isInlineConstant(DefMO, MI.getDesc().OpInfo[OpIdx]);
 648   }
 649
 650   /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
 651   /// immediate.
 652   bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
 653     const MachineOperand &MO = MI.getOperand(OpIdx);
 654     return isInlineConstant(MO, MI.getDesc().OpInfo[OpIdx].OperandType);
 655   }
 656
 657   bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
 658                         const MachineOperand &MO) const {
 659     if (!MI.getDesc().OpInfo || OpIdx >= MI.getDesc().NumOperands)
 660       return false;
 661
 662     if (MI.isCopy()) {
 663       unsigned Size = getOpSize(MI, OpIdx);
 664       assert(Size == 8 || Size == 4);
 665
 666       uint8_t OpType = (Size == 8) ?
 667         AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
 668       return isInlineConstant(MO, OpType);
 669     }
 670
 671     return isInlineConstant(MO, MI.getDesc().OpInfo[OpIdx].OperandType);
 672   }
 673
 674   bool isInlineConstant(const MachineOperand &MO) const {
 675     const MachineInstr *Parent = MO.getParent();
 676     return isInlineConstant(*Parent, Parent->getOperandNo(&MO));
 677   }
 678
 679   bool isLiteralConstant(const MachineOperand &MO,
 680                          const MCOperandInfo &OpInfo) const {
 681     return MO.isImm() && !isInlineConstant(MO, OpInfo.OperandType);
 682   }
 683
 684   bool isLiteralConstant(const MachineInstr &MI, int OpIdx) const {
 685     const MachineOperand &MO = MI.getOperand(OpIdx);
 686     return MO.isImm() && !isInlineConstant(MI, OpIdx);
 687   }
 688
 689   // Returns true if this operand could potentially require a 32-bit literal
 690   // operand, but not necessarily. A FrameIndex for example could resolve to an
 691   // inline immediate value that will not require an additional 4-bytes; this
 692   // assumes that it will.
 693   bool isLiteralConstantLike(const MachineOperand &MO,
 694                              const MCOperandInfo &OpInfo) const;
 695
 696   bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
 697                          const MachineOperand &MO) const;
 698
 699   /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
 700   /// This function will return false if you pass it a 32-bit instruction.
 701   bool hasVALU32BitEncoding(unsigned Opcode) const;
 702
 703   /// Returns true if this operand uses the constant bus.
 704   bool usesConstantBus(const MachineRegisterInfo &MRI,
 705                        const MachineOperand &MO,
 706                        const MCOperandInfo &OpInfo) const;
 707
 708   /// Return true if this instruction has any modifiers.
 709   ///  e.g. src[012]_mod, omod, clamp.
 710   bool hasModifiers(unsigned Opcode) const;
 711
 712   bool hasModifiersSet(const MachineInstr &MI,
 713                        unsigned OpName) const;
 714   bool hasAnyModifiersSet(const MachineInstr &MI) const;
 715
 716   bool canShrink(const MachineInstr &MI,
 717                  const MachineRegisterInfo &MRI) const;
 718
 719   MachineInstr *buildShrunkInst(MachineInstr &MI,
 720                                 unsigned NewOpcode) const;
 721
 722   bool verifyInstruction(const MachineInstr &MI,
 723                          StringRef &ErrInfo) const override;
 724
 725   unsigned getVALUOp(const MachineInstr &MI) const;
 726
 727   /// Return the correct register class for \p OpNo.  For target-specific
 728   /// instructions, this will return the register class that has been defined
 729   /// in tablegen.  For generic instructions, like REG_SEQUENCE it will return
 730   /// the register class of its machine operand.
 731   /// to infer the correct register class base on the other operands.
 732   const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
 733                                            unsigned OpNo) const;
 734
 735   /// Return the size in bytes of the operand OpNo on the given
 736   // instruction opcode.
 737   unsigned getOpSize(uint16_t Opcode, unsigned OpNo) const {
 738     const MCOperandInfo &OpInfo = get(Opcode).OpInfo[OpNo];
 739
 740     if (OpInfo.RegClass == -1) {
 741       // If this is an immediate operand, this must be a 32-bit literal.
 742       assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
 743       return 4;
 744     }
 745
 746     return RI.getRegSizeInBits(*RI.getRegClass(OpInfo.RegClass)) / 8;
 747   }
 748
 749   /// This form should usually be preferred since it handles operands
 750   /// with unknown register classes.
 751   unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
 752     const MachineOperand &MO = MI.getOperand(OpNo);
 753     if (MO.isReg()) {
 754       if (unsigned SubReg = MO.getSubReg()) {
 755         assert(RI.getRegSizeInBits(*RI.getSubClassWithSubReg(
 756                                    MI.getParent()->getParent()->getRegInfo().
 757                                      getRegClass(MO.getReg()), SubReg)) >= 32 &&
 758                "Sub-dword subregs are not supported");
 759         return RI.getSubRegIndexLaneMask(SubReg).getNumLanes() * 4;
 760       }
 761     }
 762     return RI.getRegSizeInBits(*getOpRegClass(MI, OpNo)) / 8;
 763   }
 764
 765   /// Legalize the \p OpIndex operand of this instruction by inserting
 766   /// a MOV.  For example:
 767   /// ADD_I32_e32 VGPR0, 15
 768   /// to
 769   /// MOV VGPR1, 15
 770   /// ADD_I32_e32 VGPR0, VGPR1
 771   ///
 772   /// If the operand being legalized is a register, then a COPY will be used
 773   /// instead of MOV.
 774   void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
 775
 776   /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
 777   /// for \p MI.
 778   bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
 779                       const MachineOperand *MO = nullptr) const;
 780
 781   /// Check if \p MO would be a valid operand for the given operand
 782   /// definition \p OpInfo. Note this does not attempt to validate constant bus
 783   /// restrictions (e.g. literal constant usage).
 784   bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
 785                           const MCOperandInfo &OpInfo,
 786                           const MachineOperand &MO) const;
 787
 788   /// Check if \p MO (a register operand) is a legal register for the
 789   /// given operand description.
 790   bool isLegalRegOperand(const MachineRegisterInfo &MRI,
 791                          const MCOperandInfo &OpInfo,
 792                          const MachineOperand &MO) const;
 793
 794   /// Legalize operands in \p MI by either commuting it or inserting a
 795   /// copy of src1.
 796   void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 797
 798   /// Fix operands in \p MI to satisfy constant bus requirements.
 799   void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 800
 801   /// Copy a value from a VGPR (\p SrcReg) to SGPR.  This function can only
 802   /// be used when it is know that the value in SrcReg is same across all
 803   /// threads in the wave.
 804   /// \returns The SGPR register that \p SrcReg was copied to.
 805   unsigned readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
 806                               MachineRegisterInfo &MRI) const;
 807
 808   void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 809
 810   void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
 811                               MachineBasicBlock::iterator I,
 812                               const TargetRegisterClass *DstRC,
 813                               MachineOperand &Op, MachineRegisterInfo &MRI,
 814                               const DebugLoc &DL) const;
 815
 816   /// Legalize all operands in this instruction.  This function may create new
 817   /// instructions and control-flow around \p MI.  If present, \p MDT is
 818   /// updated.
 819   void legalizeOperands(MachineInstr &MI,
 820                         MachineDominatorTree *MDT = nullptr) const;
 821
 822   /// Replace this instruction's opcode with the equivalent VALU
 823   /// opcode.  This function will also move the users of \p MI to the
 824   /// VALU if necessary. If present, \p MDT is updated.
 825   void moveToVALU(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
 826
 827   void insertWaitStates(MachineBasicBlock &MBB,MachineBasicBlock::iterator MI,
 828                         int Count) const;
 829
 830   void insertNoop(MachineBasicBlock &MBB,
 831                   MachineBasicBlock::iterator MI) const override;
 832
 833   void insertReturn(MachineBasicBlock &MBB) const;
 834   /// Return the number of wait states that result from executing this
 835   /// instruction.
 836   static unsigned getNumWaitStates(const MachineInstr &MI);
 837
 838   /// Returns the operand named \p Op.  If \p MI does not have an
 839   /// operand named \c Op, this function returns nullptr.
 840   LLVM_READONLY
 841   MachineOperand *getNamedOperand(MachineInstr &MI, unsigned OperandName) const;
 842
 843   LLVM_READONLY
 844   const MachineOperand *getNamedOperand(const MachineInstr &MI,
 845                                         unsigned OpName) const {
 846     return getNamedOperand(const_cast<MachineInstr &>(MI), OpName);
 847   }
 848
 849   /// Get required immediate operand
 850   int64_t getNamedImmOperand(const MachineInstr &MI, unsigned OpName) const {
 851     int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
 852     return MI.getOperand(Idx).getImm();
 853   }
 854
 855   uint64_t getDefaultRsrcDataFormat() const;
 856   uint64_t getScratchRsrcWords23() const;
 857
 858   bool isLowLatencyInstruction(const MachineInstr &MI) const;
 859   bool isHighLatencyInstruction(const MachineInstr &MI) const;
 860
 861   /// Return the descriptor of the target-specific machine instruction
 862   /// that corresponds to the specified pseudo or native opcode.
 863   const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
 864     return get(pseudoToMCOpcode(Opcode));
 865   }
 866
 867   unsigned isStackAccess(const MachineInstr &MI, int &FrameIndex) const;
 868   unsigned isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;
 869
 870   unsigned isLoadFromStackSlot(const MachineInstr &MI,
 871                                int &FrameIndex) const override;
 872   unsigned isStoreToStackSlot(const MachineInstr &MI,
 873                               int &FrameIndex) const override;
 874
 875   unsigned getInstBundleSize(const MachineInstr &MI) const;
 876   unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
 877
 878   bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
 879
 880   bool isNonUniformBranchInstr(MachineInstr &Instr) const;
 881
 882   void convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
 883                                  MachineBasicBlock *IfEnd) const;
 884
 885   void convertNonUniformLoopRegion(MachineBasicBlock *LoopEntry,
 886                                    MachineBasicBlock *LoopEnd) const;
 887
 888   std::pair<unsigned, unsigned>
 889   decomposeMachineOperandsTargetFlags(unsigned TF) const override;
 890
 891   ArrayRef<std::pair<int, const char *>>
 892   getSerializableTargetIndices() const override;
 893
 894   ArrayRef<std::pair<unsigned, const char *>>
 895   getSerializableDirectMachineOperandTargetFlags() const override;
 896
 897   ScheduleHazardRecognizer *
 898   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
 899                                  const ScheduleDAG *DAG) const override;
 900
 901   ScheduleHazardRecognizer *
 902   CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;
 903
 904   bool isBasicBlockPrologue(const MachineInstr &MI) const override;
 905
 906   /// Return a partially built integer add instruction without carry.
 907   /// Caller must add source operands.
 908   /// For pre-GFX9 it will generate unused carry destination operand.
 909   /// TODO: After GFX9 it should return a no-carry operation.
 910   MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
 911                                     MachineBasicBlock::iterator I,
 912                                     const DebugLoc &DL,
 913                                     unsigned DestReg) const;
 914
 915   static bool isKillTerminator(unsigned Opcode);
 916   const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
 917
 918   static bool isLegalMUBUFImmOffset(unsigned Imm) {
 919     return isUInt<12>(Imm);
 920   }
 921
 922   /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
 923   /// Return -1 if the target-specific opcode for the pseudo instruction does
 924   /// not exist. If Opcode is not a pseudo instruction, this is identity.
 925   int pseudoToMCOpcode(int Opcode) const;
 926 };
 927
 928 /// \brief Returns true if a reg:subreg pair P has a TRC class
 929 inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
 930                          const TargetRegisterClass &TRC,
 931                          MachineRegisterInfo &MRI) {
 932   auto *RC = MRI.getRegClass(P.Reg);
 933   if (!P.SubReg)
 934     return RC == &TRC;
 935   auto *TRI = MRI.getTargetRegisterInfo();
 936   return RC == TRI->getMatchingSuperRegClass(RC, &TRC, P.SubReg);
 937 }
 938
 939 /// \brief Create RegSubRegPair from a register MachineOperand
 940 inline
 941 TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
 942   assert(O.isReg());
 943   return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
 944 }
 945
 946 /// \brief Return the SubReg component from REG_SEQUENCE
 947 TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
 948                                                     unsigned SubReg);
 949
 950 /// \brief Return the defining instruction for a given reg:subreg pair
 951 /// skipping copy like instructions and subreg-manipulation pseudos.
 952 /// Following another subreg of a reg:subreg isn't supported.
 953 MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
 954                                MachineRegisterInfo &MRI);
 955
 956 /// \brief Return true if EXEC mask isnt' changed between the def and
 957 /// all uses of VReg. Currently if def and uses are in different BBs -
 958 /// simply return false. Should be run on SSA.
 959 bool isEXECMaskConstantBetweenDefAndUses(unsigned VReg,
 960                                          MachineRegisterInfo &MRI);
 961
 962 namespace AMDGPU {
 963
 964   LLVM_READONLY
 965   int getVOPe64(uint16_t Opcode);
 966
 967   LLVM_READONLY
 968   int getVOPe32(uint16_t Opcode);
 969
 970   LLVM_READONLY
 971   int getSDWAOp(uint16_t Opcode);
 972
 973   LLVM_READONLY
 974   int getDPPOp32(uint16_t Opcode);
 975
 976   LLVM_READONLY
 977   int getBasicFromSDWAOp(uint16_t Opcode);
 978
 979   LLVM_READONLY
 980   int getCommuteRev(uint16_t Opcode);
 981
 982   LLVM_READONLY
 983   int getCommuteOrig(uint16_t Opcode);
 984
 985   LLVM_READONLY
 986   int getAddr64Inst(uint16_t Opcode);
 987
 988   /// Check if \p Opcode is an Addr64 opcode.
 989   ///
 990   /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
 991   LLVM_READONLY
 992   int getIfAddr64Inst(uint16_t Opcode);
 993
 994   LLVM_READONLY
 995   int getMUBUFNoLdsInst(uint16_t Opcode);
 996
 997   LLVM_READONLY
 998   int getAtomicRetOp(uint16_t Opcode);
 999
1000   LLVM_READONLY
1001   int getAtomicNoRetOp(uint16_t Opcode);
1002
1003   LLVM_READONLY
1004   int getSOPKOp(uint16_t Opcode);
1005
1006   LLVM_READONLY
1007   int getGlobalSaddrOp(uint16_t Opcode);
1008
1009   const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1010   const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1011   const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1012   const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1013
1014   // For MachineOperands.
1015   enum TargetFlags {
1016     TF_LONG_BRANCH_FORWARD = 1 << 0,
1017     TF_LONG_BRANCH_BACKWARD = 1 << 1
1018   };
1019
1020 } // end namespace AMDGPU
1021
1022 namespace SI {
1023 namespace KernelInputOffsets {
1024
1025 /// Offsets in bytes from the start of the input buffer
1026 enum Offsets {
1027   NGROUPS_X = 0,
1028   NGROUPS_Y = 4,
1029   NGROUPS_Z = 8,
1030   GLOBAL_SIZE_X = 12,
1031   GLOBAL_SIZE_Y = 16,
1032   GLOBAL_SIZE_Z = 20,
1033   LOCAL_SIZE_X = 24,
1034   LOCAL_SIZE_Y = 28,
1035   LOCAL_SIZE_Z = 32
1036 };
1037
1038 } // end namespace KernelInputOffsets
1039 } // end namespace SI
1040
1041 } // end namespace llvm
1042
1043 #endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H