lib/Target/AMDGPU/AMDGPUSubtarget.h

   1 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //==-----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// AMDGPU specific subclass of TargetSubtarget.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
  15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
  16
  17 #include "AMDGPU.h"
  18 #include "AMDGPUCallLowering.h"
  19 #include "R600FrameLowering.h"
  20 #include "R600ISelLowering.h"
  21 #include "R600InstrInfo.h"
  22 #include "SIFrameLowering.h"
  23 #include "SIISelLowering.h"
  24 #include "SIInstrInfo.h"
  25 #include "Utils/AMDGPUBaseInfo.h"
  26 #include "llvm/ADT/Triple.h"
  27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  28 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
  29 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  30 #include "llvm/CodeGen/MachineFunction.h"
  31 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
  32 #include "llvm/MC/MCInstrItineraries.h"
  33 #include "llvm/Support/MathExtras.h"
  34 #include <cassert>
  35 #include <cstdint>
  36 #include <memory>
  37 #include <utility>
  38
  39 #define GET_SUBTARGETINFO_HEADER
  40 #include "AMDGPUGenSubtargetInfo.inc"
  41 #define GET_SUBTARGETINFO_HEADER
  42 #include "R600GenSubtargetInfo.inc"
  43
  44 namespace llvm {
  45
  46 class StringRef;
  47
  48 class AMDGPUSubtarget {
  49 public:
  50   enum Generation {
  51     R600 = 0,
  52     R700 = 1,
  53     EVERGREEN = 2,
  54     NORTHERN_ISLANDS = 3,
  55     SOUTHERN_ISLANDS = 4,
  56     SEA_ISLANDS = 5,
  57     VOLCANIC_ISLANDS = 6,
  58     GFX9 = 7,
  59     GFX10 = 8
  60   };
  61
  62 private:
  63   Triple TargetTriple;
  64
  65 protected:
  66   bool Has16BitInsts;
  67   bool HasMadMixInsts;
  68   bool FP32Denormals;
  69   bool FPExceptions;
  70   bool HasSDWA;
  71   bool HasVOP3PInsts;
  72   bool HasMulI24;
  73   bool HasMulU24;
  74   bool HasInv2PiInlineImm;
  75   bool HasFminFmaxLegacy;
  76   bool EnablePromoteAlloca;
  77   bool HasTrigReducedRange;
  78   unsigned MaxWavesPerEU;
  79   int LocalMemorySize;
  80   unsigned WavefrontSize;
  81
  82 public:
  83   AMDGPUSubtarget(const Triple &TT);
  84
  85   static const AMDGPUSubtarget &get(const MachineFunction &MF);
  86   static const AMDGPUSubtarget &get(const TargetMachine &TM,
  87                                     const Function &F);
  88
  89   /// \returns Default range flat work group size for a calling convention.
  90   std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
  91
  92   /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
  93   /// for function \p F, or minimum/maximum flat work group sizes explicitly
  94   /// requested using "amdgpu-flat-work-group-size" attribute attached to
  95   /// function \p F.
  96   ///
  97   /// \returns Subtarget's default values if explicitly requested values cannot
  98   /// be converted to integer, or violate subtarget's specifications.
  99   std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
 100
 101   /// \returns Subtarget's default pair of minimum/maximum number of waves per
 102   /// execution unit for function \p F, or minimum/maximum number of waves per
 103   /// execution unit explicitly requested using "amdgpu-waves-per-eu" attribute
 104   /// attached to function \p F.
 105   ///
 106   /// \returns Subtarget's default values if explicitly requested values cannot
 107   /// be converted to integer, violate subtarget's specifications, or are not
 108   /// compatible with minimum/maximum number of waves limited by flat work group
 109   /// size, register usage, and/or lds usage.
 110   std::pair<unsigned, unsigned> getWavesPerEU(const Function &F) const;
 111
 112   /// Return the amount of LDS that can be used that will not restrict the
 113   /// occupancy lower than WaveCount.
 114   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
 115                                            const Function &) const;
 116
 117   /// Inverse of getMaxLocalMemWithWaveCount. Return the maximum wavecount if
 118   /// the given LDS memory size is the only constraint.
 119   unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const;
 120
 121   unsigned getOccupancyWithLocalMemSize(const MachineFunction &MF) const;
 122
 123   bool isAmdHsaOS() const {
 124     return TargetTriple.getOS() == Triple::AMDHSA;
 125   }
 126
 127   bool isAmdPalOS() const {
 128     return TargetTriple.getOS() == Triple::AMDPAL;
 129   }
 130
 131   bool isMesa3DOS() const {
 132     return TargetTriple.getOS() == Triple::Mesa3D;
 133   }
 134
 135   bool isMesaKernel(const Function &F) const {
 136     return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
 137   }
 138
 139   bool isAmdHsaOrMesa(const Function &F) const {
 140     return isAmdHsaOS() || isMesaKernel(F);
 141   }
 142
 143   bool has16BitInsts() const {
 144     return Has16BitInsts;
 145   }
 146
 147   bool hasMadMixInsts() const {
 148     return HasMadMixInsts;
 149   }
 150
 151   bool hasFP32Denormals() const {
 152     return FP32Denormals;
 153   }
 154
 155   bool hasFPExceptions() const {
 156     return FPExceptions;
 157   }
 158
 159   bool hasSDWA() const {
 160     return HasSDWA;
 161   }
 162
 163   bool hasVOP3PInsts() const {
 164     return HasVOP3PInsts;
 165   }
 166
 167   bool hasMulI24() const {
 168     return HasMulI24;
 169   }
 170
 171   bool hasMulU24() const {
 172     return HasMulU24;
 173   }
 174
 175   bool hasInv2PiInlineImm() const {
 176     return HasInv2PiInlineImm;
 177   }
 178
 179   bool hasFminFmaxLegacy() const {
 180     return HasFminFmaxLegacy;
 181   }
 182
 183   bool hasTrigReducedRange() const {
 184     return HasTrigReducedRange;
 185   }
 186
 187   bool isPromoteAllocaEnabled() const {
 188     return EnablePromoteAlloca;
 189   }
 190
 191   unsigned getWavefrontSize() const {
 192     return WavefrontSize;
 193   }
 194
 195   int getLocalMemorySize() const {
 196     return LocalMemorySize;
 197   }
 198
 199   Align getAlignmentForImplicitArgPtr() const {
 200     return isAmdHsaOS() ? Align(8) : Align(4);
 201   }
 202
 203   /// Returns the offset in bytes from the start of the input buffer
 204   ///        of the first explicit kernel argument.
 205   unsigned getExplicitKernelArgOffset(const Function &F) const {
 206     return isAmdHsaOrMesa(F) ? 0 : 36;
 207   }
 208
 209   /// \returns Maximum number of work groups per compute unit supported by the
 210   /// subtarget and limited by given \p FlatWorkGroupSize.
 211   virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const = 0;
 212
 213   /// \returns Minimum flat work group size supported by the subtarget.
 214   virtual unsigned getMinFlatWorkGroupSize() const = 0;
 215
 216   /// \returns Maximum flat work group size supported by the subtarget.
 217   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
 218
 219   /// \returns Maximum number of waves per execution unit supported by the
 220   /// subtarget and limited by given \p FlatWorkGroupSize.
 221   virtual unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const  = 0;
 222
 223   /// \returns Minimum number of waves per execution unit supported by the
 224   /// subtarget.
 225   virtual unsigned getMinWavesPerEU() const = 0;
 226
 227   /// \returns Maximum number of waves per execution unit supported by the
 228   /// subtarget without any kind of limitation.
 229   unsigned getMaxWavesPerEU() const { return MaxWavesPerEU; }
 230
 231   /// Creates value range metadata on an workitemid.* inrinsic call or load.
 232   bool makeLIDRangeMetadata(Instruction *I) const;
 233
 234   /// \returns Number of bytes of arguments that are passed to a shader or
 235   /// kernel in addition to the explicit ones declared for the function.
 236   unsigned getImplicitArgNumBytes(const Function &F) const {
 237     if (isMesaKernel(F))
 238       return 16;
 239     return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
 240   }
 241   uint64_t getExplicitKernArgSize(const Function &F, Align &MaxAlign) const;
 242   unsigned getKernArgSegmentSize(const Function &F, Align &MaxAlign) const;
 243
 244   virtual ~AMDGPUSubtarget() {}
 245 };
 246
 247 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
 248                      public AMDGPUSubtarget {
 249
 250   using AMDGPUSubtarget::getMaxWavesPerEU;
 251
 252 public:
 253   enum TrapHandlerAbi {
 254     TrapHandlerAbiNone = 0,
 255     TrapHandlerAbiHsa = 1
 256   };
 257
 258   enum TrapID {
 259     TrapIDHardwareReserved = 0,
 260     TrapIDHSADebugTrap = 1,
 261     TrapIDLLVMTrap = 2,
 262     TrapIDLLVMDebugTrap = 3,
 263     TrapIDDebugBreakpoint = 7,
 264     TrapIDDebugReserved8 = 8,
 265     TrapIDDebugReservedFE = 0xfe,
 266     TrapIDDebugReservedFF = 0xff
 267   };
 268
 269   enum TrapRegValues {
 270     LLVMTrapHandlerRegValue = 1
 271   };
 272
 273 private:
 274   /// GlobalISel related APIs.
 275   std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
 276   std::unique_ptr<InstructionSelector> InstSelector;
 277   std::unique_ptr<LegalizerInfo> Legalizer;
 278   std::unique_ptr<RegisterBankInfo> RegBankInfo;
 279
 280 protected:
 281   // Basic subtarget description.
 282   Triple TargetTriple;
 283   unsigned Gen;
 284   InstrItineraryData InstrItins;
 285   int LDSBankCount;
 286   unsigned MaxPrivateElementSize;
 287
 288   // Possibly statically set by tablegen, but may want to be overridden.
 289   bool FastFMAF32;
 290   bool HalfRate64Ops;
 291
 292   // Dynamially set bits that enable features.
 293   bool FP64FP16Denormals;
 294   bool FlatForGlobal;
 295   bool AutoWaitcntBeforeBarrier;
 296   bool CodeObjectV3;
 297   bool UnalignedScratchAccess;
 298   bool UnalignedBufferAccess;
 299   bool HasApertureRegs;
 300   bool EnableXNACK;
 301   bool DoesNotSupportXNACK;
 302   bool EnableCuMode;
 303   bool TrapHandler;
 304
 305   // Used as options.
 306   bool EnableLoadStoreOpt;
 307   bool EnableUnsafeDSOffsetFolding;
 308   bool EnableSIScheduler;
 309   bool EnableDS128;
 310   bool EnablePRTStrictNull;
 311   bool DumpCode;
 312
 313   // Subtarget statically properties set by tablegen
 314   bool FP64;
 315   bool FMA;
 316   bool MIMG_R128;
 317   bool IsGCN;
 318   bool GCN3Encoding;
 319   bool CIInsts;
 320   bool GFX8Insts;
 321   bool GFX9Insts;
 322   bool GFX10Insts;
 323   bool GFX7GFX8GFX9Insts;
 324   bool SGPRInitBug;
 325   bool HasSMemRealTime;
 326   bool HasIntClamp;
 327   bool HasFmaMixInsts;
 328   bool HasMovrel;
 329   bool HasVGPRIndexMode;
 330   bool HasScalarStores;
 331   bool HasScalarAtomics;
 332   bool HasSDWAOmod;
 333   bool HasSDWAScalar;
 334   bool HasSDWASdst;
 335   bool HasSDWAMac;
 336   bool HasSDWAOutModsVOPC;
 337   bool HasDPP;
 338   bool HasDPP8;
 339   bool HasR128A16;
 340   bool HasNSAEncoding;
 341   bool HasDLInsts;
 342   bool HasDot1Insts;
 343   bool HasDot2Insts;
 344   bool HasDot3Insts;
 345   bool HasDot4Insts;
 346   bool HasDot5Insts;
 347   bool HasDot6Insts;
 348   bool HasMAIInsts;
 349   bool HasPkFmacF16Inst;
 350   bool HasAtomicFaddInsts;
 351   bool EnableSRAMECC;
 352   bool DoesNotSupportSRAMECC;
 353   bool HasNoSdstCMPX;
 354   bool HasVscnt;
 355   bool HasRegisterBanking;
 356   bool HasVOP3Literal;
 357   bool HasNoDataDepHazard;
 358   bool FlatAddressSpace;
 359   bool FlatInstOffsets;
 360   bool FlatGlobalInsts;
 361   bool FlatScratchInsts;
 362   bool ScalarFlatScratchInsts;
 363   bool AddNoCarryInsts;
 364   bool HasUnpackedD16VMem;
 365   bool R600ALUInst;
 366   bool CaymanISA;
 367   bool CFALUBug;
 368   bool LDSMisalignedBug;
 369   bool HasMFMAInlineLiteralBug;
 370   bool HasVertexCache;
 371   short TexVTXClauseSize;
 372   bool ScalarizeGlobal;
 373
 374   bool HasVcmpxPermlaneHazard;
 375   bool HasVMEMtoScalarWriteHazard;
 376   bool HasSMEMtoVectorWriteHazard;
 377   bool HasInstFwdPrefetchBug;
 378   bool HasVcmpxExecWARHazard;
 379   bool HasLdsBranchVmemWARHazard;
 380   bool HasNSAtoVMEMBug;
 381   bool HasOffset3fBug;
 382   bool HasFlatSegmentOffsetBug;
 383
 384   // Dummy feature to use for assembler in tablegen.
 385   bool FeatureDisable;
 386
 387   SelectionDAGTargetInfo TSInfo;
 388 private:
 389   SIInstrInfo InstrInfo;
 390   SITargetLowering TLInfo;
 391   SIFrameLowering FrameLowering;
 392
 393   // See COMPUTE_TMPRING_SIZE.WAVESIZE, 13-bit field in units of 256-dword.
 394   static const unsigned MaxWaveScratchSize = (256 * 4) * ((1 << 13) - 1);
 395
 396 public:
 397   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
 398                const GCNTargetMachine &TM);
 399   ~GCNSubtarget() override;
 400
 401   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
 402                                                    StringRef GPU, StringRef FS);
 403
 404   const SIInstrInfo *getInstrInfo() const override {
 405     return &InstrInfo;
 406   }
 407
 408   const SIFrameLowering *getFrameLowering() const override {
 409     return &FrameLowering;
 410   }
 411
 412   const SITargetLowering *getTargetLowering() const override {
 413     return &TLInfo;
 414   }
 415
 416   const SIRegisterInfo *getRegisterInfo() const override {
 417     return &InstrInfo.getRegisterInfo();
 418   }
 419
 420   const CallLowering *getCallLowering() const override {
 421     return CallLoweringInfo.get();
 422   }
 423
 424   InstructionSelector *getInstructionSelector() const override {
 425     return InstSelector.get();
 426   }
 427
 428   const LegalizerInfo *getLegalizerInfo() const override {
 429     return Legalizer.get();
 430   }
 431
 432   const RegisterBankInfo *getRegBankInfo() const override {
 433     return RegBankInfo.get();
 434   }
 435
 436   // Nothing implemented, just prevent crashes on use.
 437   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
 438     return &TSInfo;
 439   }
 440
 441   const InstrItineraryData *getInstrItineraryData() const override {
 442     return &InstrItins;
 443   }
 444
 445   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
 446
 447   Generation getGeneration() const {
 448     return (Generation)Gen;
 449   }
 450
 451   unsigned getWavefrontSizeLog2() const {
 452     return Log2_32(WavefrontSize);
 453   }
 454
 455   /// Return the number of high bits known to be zero fror a frame index.
 456   unsigned getKnownHighZeroBitsForFrameIndex() const {
 457     return countLeadingZeros(MaxWaveScratchSize) + getWavefrontSizeLog2();
 458   }
 459
 460   int getLDSBankCount() const {
 461     return LDSBankCount;
 462   }
 463
 464   unsigned getMaxPrivateElementSize() const {
 465     return MaxPrivateElementSize;
 466   }
 467
 468   unsigned getConstantBusLimit(unsigned Opcode) const;
 469
 470   bool hasIntClamp() const {
 471     return HasIntClamp;
 472   }
 473
 474   bool hasFP64() const {
 475     return FP64;
 476   }
 477
 478   bool hasMIMG_R128() const {
 479     return MIMG_R128;
 480   }
 481
 482   bool hasHWFP64() const {
 483     return FP64;
 484   }
 485
 486   bool hasFastFMAF32() const {
 487     return FastFMAF32;
 488   }
 489
 490   bool hasHalfRate64Ops() const {
 491     return HalfRate64Ops;
 492   }
 493
 494   bool hasAddr64() const {
 495     return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);
 496   }
 497
 498   // Return true if the target only has the reverse operand versions of VALU
 499   // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).
 500   bool hasOnlyRevVALUShifts() const {
 501     return getGeneration() >= VOLCANIC_ISLANDS;
 502   }
 503
 504   bool hasBFE() const {
 505     return true;
 506   }
 507
 508   bool hasBFI() const {
 509     return true;
 510   }
 511
 512   bool hasBFM() const {
 513     return hasBFE();
 514   }
 515
 516   bool hasBCNT(unsigned Size) const {
 517     return true;
 518   }
 519
 520   bool hasFFBL() const {
 521     return true;
 522   }
 523
 524   bool hasFFBH() const {
 525     return true;
 526   }
 527
 528   bool hasMed3_16() const {
 529     return getGeneration() >= AMDGPUSubtarget::GFX9;
 530   }
 531
 532   bool hasMin3Max3_16() const {
 533     return getGeneration() >= AMDGPUSubtarget::GFX9;
 534   }
 535
 536   bool hasFmaMixInsts() const {
 537     return HasFmaMixInsts;
 538   }
 539
 540   bool hasCARRY() const {
 541     return true;
 542   }
 543
 544   bool hasFMA() const {
 545     return FMA;
 546   }
 547
 548   bool hasSwap() const {
 549     return GFX9Insts;
 550   }
 551
 552   bool hasScalarPackInsts() const {
 553     return GFX9Insts;
 554   }
 555
 556   bool hasScalarMulHiInsts() const {
 557     return GFX9Insts;
 558   }
 559
 560   TrapHandlerAbi getTrapHandlerAbi() const {
 561     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
 562   }
 563
 564   /// True if the offset field of DS instructions works as expected. On SI, the
 565   /// offset uses a 16-bit adder and does not always wrap properly.
 566   bool hasUsableDSOffset() const {
 567     return getGeneration() >= SEA_ISLANDS;
 568   }
 569
 570   bool unsafeDSOffsetFoldingEnabled() const {
 571     return EnableUnsafeDSOffsetFolding;
 572   }
 573
 574   /// Condition output from div_scale is usable.
 575   bool hasUsableDivScaleConditionOutput() const {
 576     return getGeneration() != SOUTHERN_ISLANDS;
 577   }
 578
 579   /// Extra wait hazard is needed in some cases before
 580   /// s_cbranch_vccnz/s_cbranch_vccz.
 581   bool hasReadVCCZBug() const {
 582     return getGeneration() <= SEA_ISLANDS;
 583   }
 584
 585   /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR
 586   /// was written by a VALU instruction.
 587   bool hasSMRDReadVALUDefHazard() const {
 588     return getGeneration() == SOUTHERN_ISLANDS;
 589   }
 590
 591   /// A read of an SGPR by a VMEM instruction requires 5 wait states when the
 592   /// SGPR was written by a VALU Instruction.
 593   bool hasVMEMReadSGPRVALUDefHazard() const {
 594     return getGeneration() >= VOLCANIC_ISLANDS;
 595   }
 596
 597   bool hasRFEHazards() const {
 598     return getGeneration() >= VOLCANIC_ISLANDS;
 599   }
 600
 601   /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
 602   unsigned getSetRegWaitStates() const {
 603     return getGeneration() <= SEA_ISLANDS ? 1 : 2;
 604   }
 605
 606   bool dumpCode() const {
 607     return DumpCode;
 608   }
 609
 610   /// Return the amount of LDS that can be used that will not restrict the
 611   /// occupancy lower than WaveCount.
 612   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
 613                                            const Function &) const;
 614
 615   bool hasFP16Denormals() const {
 616     return FP64FP16Denormals;
 617   }
 618
 619   bool hasFP64Denormals() const {
 620     return FP64FP16Denormals;
 621   }
 622
 623   bool supportsMinMaxDenormModes() const {
 624     return getGeneration() >= AMDGPUSubtarget::GFX9;
 625   }
 626
 627   /// \returns If target supports S_DENORM_MODE.
 628   bool hasDenormModeInst() const {
 629     return getGeneration() >= AMDGPUSubtarget::GFX10;
 630   }
 631
 632   bool useFlatForGlobal() const {
 633     return FlatForGlobal;
 634   }
 635
 636   /// \returns If target supports ds_read/write_b128 and user enables generation
 637   /// of ds_read/write_b128.
 638   bool useDS128() const {
 639     return CIInsts && EnableDS128;
 640   }
 641
 642   /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64
 643   bool haveRoundOpsF64() const {
 644     return CIInsts;
 645   }
 646
 647   /// \returns If MUBUF instructions always perform range checking, even for
 648   /// buffer resources used for private memory access.
 649   bool privateMemoryResourceIsRangeChecked() const {
 650     return getGeneration() < AMDGPUSubtarget::GFX9;
 651   }
 652
 653   /// \returns If target requires PRT Struct NULL support (zero result registers
 654   /// for sparse texture support).
 655   bool usePRTStrictNull() const {
 656     return EnablePRTStrictNull;
 657   }
 658
 659   bool hasAutoWaitcntBeforeBarrier() const {
 660     return AutoWaitcntBeforeBarrier;
 661   }
 662
 663   bool hasCodeObjectV3() const {
 664     // FIXME: Need to add code object v3 support for mesa and pal.
 665     return isAmdHsaOS() ? CodeObjectV3 : false;
 666   }
 667
 668   bool hasUnalignedBufferAccess() const {
 669     return UnalignedBufferAccess;
 670   }
 671
 672   bool hasUnalignedScratchAccess() const {
 673     return UnalignedScratchAccess;
 674   }
 675
 676   bool hasApertureRegs() const {
 677     return HasApertureRegs;
 678   }
 679
 680   bool isTrapHandlerEnabled() const {
 681     return TrapHandler;
 682   }
 683
 684   bool isXNACKEnabled() const {
 685     return EnableXNACK;
 686   }
 687
 688   bool isCuModeEnabled() const {
 689     return EnableCuMode;
 690   }
 691
 692   bool hasFlatAddressSpace() const {
 693     return FlatAddressSpace;
 694   }
 695
 696   bool hasFlatScrRegister() const {
 697     return hasFlatAddressSpace();
 698   }
 699
 700   bool hasFlatInstOffsets() const {
 701     return FlatInstOffsets;
 702   }
 703
 704   bool hasFlatGlobalInsts() const {
 705     return FlatGlobalInsts;
 706   }
 707
 708   bool hasFlatScratchInsts() const {
 709     return FlatScratchInsts;
 710   }
 711
 712   bool hasScalarFlatScratchInsts() const {
 713     return ScalarFlatScratchInsts;
 714   }
 715
 716   bool hasFlatSegmentOffsetBug() const {
 717     return HasFlatSegmentOffsetBug;
 718   }
 719
 720   bool hasFlatLgkmVMemCountInOrder() const {
 721     return getGeneration() > GFX9;
 722   }
 723
 724   bool hasD16LoadStore() const {
 725     return getGeneration() >= GFX9;
 726   }
 727
 728   bool d16PreservesUnusedBits() const {
 729     return hasD16LoadStore() && !isSRAMECCEnabled();
 730   }
 731
 732   bool hasD16Images() const {
 733     return getGeneration() >= VOLCANIC_ISLANDS;
 734   }
 735
 736   /// Return if most LDS instructions have an m0 use that require m0 to be
 737   /// iniitalized.
 738   bool ldsRequiresM0Init() const {
 739     return getGeneration() < GFX9;
 740   }
 741
 742   // True if the hardware rewinds and replays GWS operations if a wave is
 743   // preempted.
 744   //
 745   // If this is false, a GWS operation requires testing if a nack set the
 746   // MEM_VIOL bit, and repeating if so.
 747   bool hasGWSAutoReplay() const {
 748     return getGeneration() >= GFX9;
 749   }
 750
 751   /// \returns if target has ds_gws_sema_release_all instruction.
 752   bool hasGWSSemaReleaseAll() const {
 753     return CIInsts;
 754   }
 755
 756   bool hasAddNoCarry() const {
 757     return AddNoCarryInsts;
 758   }
 759
 760   bool hasUnpackedD16VMem() const {
 761     return HasUnpackedD16VMem;
 762   }
 763
 764   // Covers VS/PS/CS graphics shaders
 765   bool isMesaGfxShader(const Function &F) const {
 766     return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());
 767   }
 768
 769   bool hasMad64_32() const {
 770     return getGeneration() >= SEA_ISLANDS;
 771   }
 772
 773   bool hasSDWAOmod() const {
 774     return HasSDWAOmod;
 775   }
 776
 777   bool hasSDWAScalar() const {
 778     return HasSDWAScalar;
 779   }
 780
 781   bool hasSDWASdst() const {
 782     return HasSDWASdst;
 783   }
 784
 785   bool hasSDWAMac() const {
 786     return HasSDWAMac;
 787   }
 788
 789   bool hasSDWAOutModsVOPC() const {
 790     return HasSDWAOutModsVOPC;
 791   }
 792
 793   bool hasDLInsts() const {
 794     return HasDLInsts;
 795   }
 796
 797   bool hasDot1Insts() const {
 798     return HasDot1Insts;
 799   }
 800
 801   bool hasDot2Insts() const {
 802     return HasDot2Insts;
 803   }
 804
 805   bool hasDot3Insts() const {
 806     return HasDot3Insts;
 807   }
 808
 809   bool hasDot4Insts() const {
 810     return HasDot4Insts;
 811   }
 812
 813   bool hasDot5Insts() const {
 814     return HasDot5Insts;
 815   }
 816
 817   bool hasDot6Insts() const {
 818     return HasDot6Insts;
 819   }
 820
 821   bool hasMAIInsts() const {
 822     return HasMAIInsts;
 823   }
 824
 825   bool hasPkFmacF16Inst() const {
 826     return HasPkFmacF16Inst;
 827   }
 828
 829   bool hasAtomicFaddInsts() const {
 830     return HasAtomicFaddInsts;
 831   }
 832
 833   bool isSRAMECCEnabled() const {
 834     return EnableSRAMECC;
 835   }
 836
 837   bool hasNoSdstCMPX() const {
 838     return HasNoSdstCMPX;
 839   }
 840
 841   bool hasVscnt() const {
 842     return HasVscnt;
 843   }
 844
 845   bool hasRegisterBanking() const {
 846     return HasRegisterBanking;
 847   }
 848
 849   bool hasVOP3Literal() const {
 850     return HasVOP3Literal;
 851   }
 852
 853   bool hasNoDataDepHazard() const {
 854     return HasNoDataDepHazard;
 855   }
 856
 857   bool vmemWriteNeedsExpWaitcnt() const {
 858     return getGeneration() < SEA_ISLANDS;
 859   }
 860
 861   // Scratch is allocated in 256 dword per wave blocks for the entire
 862   // wavefront. When viewed from the perspecive of an arbitrary workitem, this
 863   // is 4-byte aligned.
 864   //
 865   // Only 4-byte alignment is really needed to access anything. Transformations
 866   // on the pointer value itself may rely on the alignment / known low bits of
 867   // the pointer. Set this to something above the minimum to avoid needing
 868   // dynamic realignment in common cases.
 869   unsigned getStackAlignment() const {
 870     return 16;
 871   }
 872
 873   bool enableMachineScheduler() const override {
 874     return true;
 875   }
 876
 877   bool enableSubRegLiveness() const override {
 878     return true;
 879   }
 880
 881   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
 882   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
 883
 884   /// \returns Number of execution units per compute unit supported by the
 885   /// subtarget.
 886   unsigned getEUsPerCU() const {
 887     return AMDGPU::IsaInfo::getEUsPerCU(this);
 888   }
 889
 890   /// \returns Maximum number of waves per compute unit supported by the
 891   /// subtarget without any kind of limitation.
 892   unsigned getMaxWavesPerCU() const {
 893     return AMDGPU::IsaInfo::getMaxWavesPerCU(this);
 894   }
 895
 896   /// \returns Maximum number of waves per compute unit supported by the
 897   /// subtarget and limited by given \p FlatWorkGroupSize.
 898   unsigned getMaxWavesPerCU(unsigned FlatWorkGroupSize) const {
 899     return AMDGPU::IsaInfo::getMaxWavesPerCU(this, FlatWorkGroupSize);
 900   }
 901
 902   /// \returns Number of waves per work group supported by the subtarget and
 903   /// limited by given \p FlatWorkGroupSize.
 904   unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
 905     return AMDGPU::IsaInfo::getWavesPerWorkGroup(this, FlatWorkGroupSize);
 906   }
 907
 908   // static wrappers
 909   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
 910
 911   // XXX - Why is this here if it isn't in the default pass set?
 912   bool enableEarlyIfConversion() const override {
 913     return true;
 914   }
 915
 916   void overrideSchedPolicy(MachineSchedPolicy &Policy,
 917                            unsigned NumRegionInstrs) const override;
 918
 919   unsigned getMaxNumUserSGPRs() const {
 920     return 16;
 921   }
 922
 923   bool hasSMemRealTime() const {
 924     return HasSMemRealTime;
 925   }
 926
 927   bool hasMovrel() const {
 928     return HasMovrel;
 929   }
 930
 931   bool hasVGPRIndexMode() const {
 932     return HasVGPRIndexMode;
 933   }
 934
 935   bool useVGPRIndexMode(bool UserEnable) const {
 936     return !hasMovrel() || (UserEnable && hasVGPRIndexMode());
 937   }
 938
 939   bool hasScalarCompareEq64() const {
 940     return getGeneration() >= VOLCANIC_ISLANDS;
 941   }
 942
 943   bool hasScalarStores() const {
 944     return HasScalarStores;
 945   }
 946
 947   bool hasScalarAtomics() const {
 948     return HasScalarAtomics;
 949   }
 950
 951   bool hasLDSFPAtomics() const {
 952     return GFX8Insts;
 953   }
 954
 955   bool hasDPP() const {
 956     return HasDPP;
 957   }
 958
 959   bool hasDPPBroadcasts() const {
 960     return HasDPP && getGeneration() < GFX10;
 961   }
 962
 963   bool hasDPPWavefrontShifts() const {
 964     return HasDPP && getGeneration() < GFX10;
 965   }
 966
 967   bool hasDPP8() const {
 968     return HasDPP8;
 969   }
 970
 971   bool hasR128A16() const {
 972     return HasR128A16;
 973   }
 974
 975   bool hasOffset3fBug() const {
 976     return HasOffset3fBug;
 977   }
 978
 979   bool hasNSAEncoding() const {
 980     return HasNSAEncoding;
 981   }
 982
 983   bool hasMadF16() const;
 984
 985   bool enableSIScheduler() const {
 986     return EnableSIScheduler;
 987   }
 988
 989   bool loadStoreOptEnabled() const {
 990     return EnableLoadStoreOpt;
 991   }
 992
 993   bool hasSGPRInitBug() const {
 994     return SGPRInitBug;
 995   }
 996
 997   bool hasMFMAInlineLiteralBug() const {
 998     return HasMFMAInlineLiteralBug;
 999   }
1000
1001   bool has12DWordStoreHazard() const {
1002     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
1003   }
1004
1005   // \returns true if the subtarget supports DWORDX3 load/store instructions.
1006   bool hasDwordx3LoadStores() const {
1007     return CIInsts;
1008   }
1009
1010   bool hasSMovFedHazard() const {
1011     return getGeneration() == AMDGPUSubtarget::GFX9;
1012   }
1013
1014   bool hasReadM0MovRelInterpHazard() const {
1015     return getGeneration() == AMDGPUSubtarget::GFX9;
1016   }
1017
1018   bool hasReadM0SendMsgHazard() const {
1019     return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1020            getGeneration() <= AMDGPUSubtarget::GFX9;
1021   }
1022
1023   bool hasVcmpxPermlaneHazard() const {
1024     return HasVcmpxPermlaneHazard;
1025   }
1026
1027   bool hasVMEMtoScalarWriteHazard() const {
1028     return HasVMEMtoScalarWriteHazard;
1029   }
1030
1031   bool hasSMEMtoVectorWriteHazard() const {
1032     return HasSMEMtoVectorWriteHazard;
1033   }
1034
1035   bool hasLDSMisalignedBug() const {
1036     return LDSMisalignedBug && !EnableCuMode;
1037   }
1038
1039   bool hasInstFwdPrefetchBug() const {
1040     return HasInstFwdPrefetchBug;
1041   }
1042
1043   bool hasVcmpxExecWARHazard() const {
1044     return HasVcmpxExecWARHazard;
1045   }
1046
1047   bool hasLdsBranchVmemWARHazard() const {
1048     return HasLdsBranchVmemWARHazard;
1049   }
1050
1051   bool hasNSAtoVMEMBug() const {
1052     return HasNSAtoVMEMBug;
1053   }
1054
1055   /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
1056   /// SGPRs
1057   unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
1058
1059   /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
1060   /// VGPRs
1061   unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
1062
1063   /// Return occupancy for the given function. Used LDS and a number of
1064   /// registers if provided.
1065   /// Note, occupancy can be affected by the scratch allocation as well, but
1066   /// we do not have enough information to compute it.
1067   unsigned computeOccupancy(const MachineFunction &MF, unsigned LDSSize = 0,
1068                             unsigned NumSGPRs = 0, unsigned NumVGPRs = 0) const;
1069
1070   /// \returns true if the flat_scratch register should be initialized with the
1071   /// pointer to the wave's scratch memory rather than a size and offset.
1072   bool flatScratchIsPointer() const {
1073     return getGeneration() >= AMDGPUSubtarget::GFX9;
1074   }
1075
1076   /// \returns true if the machine has merged shaders in which s0-s7 are
1077   /// reserved by the hardware and user SGPRs start at s8
1078   bool hasMergedShaders() const {
1079     return getGeneration() >= GFX9;
1080   }
1081
1082   /// \returns SGPR allocation granularity supported by the subtarget.
1083   unsigned getSGPRAllocGranule() const {
1084     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
1085   }
1086
1087   /// \returns SGPR encoding granularity supported by the subtarget.
1088   unsigned getSGPREncodingGranule() const {
1089     return AMDGPU::IsaInfo::getSGPREncodingGranule(this);
1090   }
1091
1092   /// \returns Total number of SGPRs supported by the subtarget.
1093   unsigned getTotalNumSGPRs() const {
1094     return AMDGPU::IsaInfo::getTotalNumSGPRs(this);
1095   }
1096
1097   /// \returns Addressable number of SGPRs supported by the subtarget.
1098   unsigned getAddressableNumSGPRs() const {
1099     return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);
1100   }
1101
1102   /// \returns Minimum number of SGPRs that meets the given number of waves per
1103   /// execution unit requirement supported by the subtarget.
1104   unsigned getMinNumSGPRs(unsigned WavesPerEU) const {
1105     return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);
1106   }
1107
1108   /// \returns Maximum number of SGPRs that meets the given number of waves per
1109   /// execution unit requirement supported by the subtarget.
1110   unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {
1111     return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);
1112   }
1113
1114   /// \returns Reserved number of SGPRs for given function \p MF.
1115   unsigned getReservedNumSGPRs(const MachineFunction &MF) const;
1116
1117   /// \returns Maximum number of SGPRs that meets number of waves per execution
1118   /// unit requirement for function \p MF, or number of SGPRs explicitly
1119   /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.
1120   ///
1121   /// \returns Value that meets number of waves per execution unit requirement
1122   /// if explicitly requested value cannot be converted to integer, violates
1123   /// subtarget's specifications, or does not meet number of waves per execution
1124   /// unit requirement.
1125   unsigned getMaxNumSGPRs(const MachineFunction &MF) const;
1126
1127   /// \returns VGPR allocation granularity supported by the subtarget.
1128   unsigned getVGPRAllocGranule() const {
1129     return AMDGPU::IsaInfo::getVGPRAllocGranule(this);
1130   }
1131
1132   /// \returns VGPR encoding granularity supported by the subtarget.
1133   unsigned getVGPREncodingGranule() const {
1134     return AMDGPU::IsaInfo::getVGPREncodingGranule(this);
1135   }
1136
1137   /// \returns Total number of VGPRs supported by the subtarget.
1138   unsigned getTotalNumVGPRs() const {
1139     return AMDGPU::IsaInfo::getTotalNumVGPRs(this);
1140   }
1141
1142   /// \returns Addressable number of VGPRs supported by the subtarget.
1143   unsigned getAddressableNumVGPRs() const {
1144     return AMDGPU::IsaInfo::getAddressableNumVGPRs(this);
1145   }
1146
1147   /// \returns Minimum number of VGPRs that meets given number of waves per
1148   /// execution unit requirement supported by the subtarget.
1149   unsigned getMinNumVGPRs(unsigned WavesPerEU) const {
1150     return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU);
1151   }
1152
1153   /// \returns Maximum number of VGPRs that meets given number of waves per
1154   /// execution unit requirement supported by the subtarget.
1155   unsigned getMaxNumVGPRs(unsigned WavesPerEU) const {
1156     return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU);
1157   }
1158
1159   /// \returns Maximum number of VGPRs that meets number of waves per execution
1160   /// unit requirement for function \p MF, or number of VGPRs explicitly
1161   /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.
1162   ///
1163   /// \returns Value that meets number of waves per execution unit requirement
1164   /// if explicitly requested value cannot be converted to integer, violates
1165   /// subtarget's specifications, or does not meet number of waves per execution
1166   /// unit requirement.
1167   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
1168
1169   void getPostRAMutations(
1170       std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
1171       const override;
1172
1173   bool isWave32() const {
1174     return WavefrontSize == 32;
1175   }
1176
1177   const TargetRegisterClass *getBoolRC() const {
1178     return getRegisterInfo()->getBoolRC();
1179   }
1180
1181   /// \returns Maximum number of work groups per compute unit supported by the
1182   /// subtarget and limited by given \p FlatWorkGroupSize.
1183   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1184     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1185   }
1186
1187   /// \returns Minimum flat work group size supported by the subtarget.
1188   unsigned getMinFlatWorkGroupSize() const override {
1189     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1190   }
1191
1192   /// \returns Maximum flat work group size supported by the subtarget.
1193   unsigned getMaxFlatWorkGroupSize() const override {
1194     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1195   }
1196
1197   /// \returns Maximum number of waves per execution unit supported by the
1198   /// subtarget and limited by given \p FlatWorkGroupSize.
1199   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1200     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1201   }
1202
1203   /// \returns Minimum number of waves per execution unit supported by the
1204   /// subtarget.
1205   unsigned getMinWavesPerEU() const override {
1206     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1207   }
1208 };
1209
1210 class R600Subtarget final : public R600GenSubtargetInfo,
1211                             public AMDGPUSubtarget {
1212 private:
1213   R600InstrInfo InstrInfo;
1214   R600FrameLowering FrameLowering;
1215   bool FMA;
1216   bool CaymanISA;
1217   bool CFALUBug;
1218   bool HasVertexCache;
1219   bool R600ALUInst;
1220   bool FP64;
1221   short TexVTXClauseSize;
1222   Generation Gen;
1223   R600TargetLowering TLInfo;
1224   InstrItineraryData InstrItins;
1225   SelectionDAGTargetInfo TSInfo;
1226
1227 public:
1228   R600Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
1229                 const TargetMachine &TM);
1230
1231   const R600InstrInfo *getInstrInfo() const override { return &InstrInfo; }
1232
1233   const R600FrameLowering *getFrameLowering() const override {
1234     return &FrameLowering;
1235   }
1236
1237   const R600TargetLowering *getTargetLowering() const override {
1238     return &TLInfo;
1239   }
1240
1241   const R600RegisterInfo *getRegisterInfo() const override {
1242     return &InstrInfo.getRegisterInfo();
1243   }
1244
1245   const InstrItineraryData *getInstrItineraryData() const override {
1246     return &InstrItins;
1247   }
1248
1249   // Nothing implemented, just prevent crashes on use.
1250   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
1251     return &TSInfo;
1252   }
1253
1254   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
1255
1256   Generation getGeneration() const {
1257     return Gen;
1258   }
1259
1260   unsigned getStackAlignment() const {
1261     return 4;
1262   }
1263
1264   R600Subtarget &initializeSubtargetDependencies(const Triple &TT,
1265                                                  StringRef GPU, StringRef FS);
1266
1267   bool hasBFE() const {
1268     return (getGeneration() >= EVERGREEN);
1269   }
1270
1271   bool hasBFI() const {
1272     return (getGeneration() >= EVERGREEN);
1273   }
1274
1275   bool hasBCNT(unsigned Size) const {
1276     if (Size == 32)
1277       return (getGeneration() >= EVERGREEN);
1278
1279     return false;
1280   }
1281
1282   bool hasBORROW() const {
1283     return (getGeneration() >= EVERGREEN);
1284   }
1285
1286   bool hasCARRY() const {
1287     return (getGeneration() >= EVERGREEN);
1288   }
1289
1290   bool hasCaymanISA() const {
1291     return CaymanISA;
1292   }
1293
1294   bool hasFFBL() const {
1295     return (getGeneration() >= EVERGREEN);
1296   }
1297
1298   bool hasFFBH() const {
1299     return (getGeneration() >= EVERGREEN);
1300   }
1301
1302   bool hasFMA() const { return FMA; }
1303
1304   bool hasCFAluBug() const { return CFALUBug; }
1305
1306   bool hasVertexCache() const { return HasVertexCache; }
1307
1308   short getTexVTXClauseSize() const { return TexVTXClauseSize; }
1309
1310   bool enableMachineScheduler() const override {
1311     return true;
1312   }
1313
1314   bool enableSubRegLiveness() const override {
1315     return true;
1316   }
1317
1318   /// \returns Maximum number of work groups per compute unit supported by the
1319   /// subtarget and limited by given \p FlatWorkGroupSize.
1320   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1321     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1322   }
1323
1324   /// \returns Minimum flat work group size supported by the subtarget.
1325   unsigned getMinFlatWorkGroupSize() const override {
1326     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1327   }
1328
1329   /// \returns Maximum flat work group size supported by the subtarget.
1330   unsigned getMaxFlatWorkGroupSize() const override {
1331     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1332   }
1333
1334   /// \returns Maximum number of waves per execution unit supported by the
1335   /// subtarget and limited by given \p FlatWorkGroupSize.
1336   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1337     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1338   }
1339
1340   /// \returns Minimum number of waves per execution unit supported by the
1341   /// subtarget.
1342   unsigned getMinWavesPerEU() const override {
1343     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1344   }
1345 };
1346
1347 } // end namespace llvm
1348
1349 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H