lib/Target/AMDGPU/AMDGPUSubtarget.h

   1 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //==-----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// AMDGPU specific subclass of TargetSubtarget.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
  15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
  16
  17 #include "AMDGPU.h"
  18 #include "AMDGPUCallLowering.h"
  19 #include "R600FrameLowering.h"
  20 #include "R600ISelLowering.h"
  21 #include "R600InstrInfo.h"
  22 #include "SIFrameLowering.h"
  23 #include "SIISelLowering.h"
  24 #include "SIInstrInfo.h"
  25 #include "Utils/AMDGPUBaseInfo.h"
  26 #include "llvm/ADT/Triple.h"
  27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  28 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
  29 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  30 #include "llvm/CodeGen/MachineFunction.h"
  31 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
  32 #include "llvm/MC/MCInstrItineraries.h"
  33 #include "llvm/Support/MathExtras.h"
  34 #include <cassert>
  35 #include <cstdint>
  36 #include <memory>
  37 #include <utility>
  38
  39 #define GET_SUBTARGETINFO_HEADER
  40 #include "AMDGPUGenSubtargetInfo.inc"
  41 #define GET_SUBTARGETINFO_HEADER
  42 #include "R600GenSubtargetInfo.inc"
  43
  44 namespace llvm {
  45
  46 class StringRef;
  47
  48 class AMDGPUSubtarget {
  49 public:
  50   enum Generation {
  51     R600 = 0,
  52     R700 = 1,
  53     EVERGREEN = 2,
  54     NORTHERN_ISLANDS = 3,
  55     SOUTHERN_ISLANDS = 4,
  56     SEA_ISLANDS = 5,
  57     VOLCANIC_ISLANDS = 6,
  58     GFX9 = 7,
  59     GFX10 = 8
  60   };
  61
  62 private:
  63   Triple TargetTriple;
  64
  65 protected:
  66   bool Has16BitInsts;
  67   bool HasMadMixInsts;
  68   bool FP32Denormals;
  69   bool FPExceptions;
  70   bool HasSDWA;
  71   bool HasVOP3PInsts;
  72   bool HasMulI24;
  73   bool HasMulU24;
  74   bool HasInv2PiInlineImm;
  75   bool HasFminFmaxLegacy;
  76   bool EnablePromoteAlloca;
  77   bool HasTrigReducedRange;
  78   unsigned MaxWavesPerEU;
  79   int LocalMemorySize;
  80   unsigned WavefrontSize;
  81
  82 public:
  83   AMDGPUSubtarget(const Triple &TT);
  84
  85   static const AMDGPUSubtarget &get(const MachineFunction &MF);
  86   static const AMDGPUSubtarget &get(const TargetMachine &TM,
  87                                     const Function &F);
  88
  89   /// \returns Default range flat work group size for a calling convention.
  90   std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
  91
  92   /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
  93   /// for function \p F, or minimum/maximum flat work group sizes explicitly
  94   /// requested using "amdgpu-flat-work-group-size" attribute attached to
  95   /// function \p F.
  96   ///
  97   /// \returns Subtarget's default values if explicitly requested values cannot
  98   /// be converted to integer, or violate subtarget's specifications.
  99   std::pair<unsigned, unsigned> getFlatWorkGroupSizes(const Function &F) const;
 100
 101   /// \returns Subtarget's default pair of minimum/maximum number of waves per
 102   /// execution unit for function \p F, or minimum/maximum number of waves per
 103   /// execution unit explicitly requested using "amdgpu-waves-per-eu" attribute
 104   /// attached to function \p F.
 105   ///
 106   /// \returns Subtarget's default values if explicitly requested values cannot
 107   /// be converted to integer, violate subtarget's specifications, or are not
 108   /// compatible with minimum/maximum number of waves limited by flat work group
 109   /// size, register usage, and/or lds usage.
 110   std::pair<unsigned, unsigned> getWavesPerEU(const Function &F) const;
 111
 112   /// Return the amount of LDS that can be used that will not restrict the
 113   /// occupancy lower than WaveCount.
 114   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
 115                                            const Function &) const;
 116
 117   /// Inverse of getMaxLocalMemWithWaveCount. Return the maximum wavecount if
 118   /// the given LDS memory size is the only constraint.
 119   unsigned getOccupancyWithLocalMemSize(uint32_t Bytes, const Function &) const;
 120
 121   unsigned getOccupancyWithLocalMemSize(const MachineFunction &MF) const;
 122
 123   bool isAmdHsaOS() const {
 124     return TargetTriple.getOS() == Triple::AMDHSA;
 125   }
 126
 127   bool isAmdPalOS() const {
 128     return TargetTriple.getOS() == Triple::AMDPAL;
 129   }
 130
 131   bool isMesa3DOS() const {
 132     return TargetTriple.getOS() == Triple::Mesa3D;
 133   }
 134
 135   bool isMesaKernel(const Function &F) const {
 136     return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
 137   }
 138
 139   bool isAmdHsaOrMesa(const Function &F) const {
 140     return isAmdHsaOS() || isMesaKernel(F);
 141   }
 142
 143   bool has16BitInsts() const {
 144     return Has16BitInsts;
 145   }
 146
 147   bool hasMadMixInsts() const {
 148     return HasMadMixInsts;
 149   }
 150
 151   bool hasFP32Denormals() const {
 152     return FP32Denormals;
 153   }
 154
 155   bool hasFPExceptions() const {
 156     return FPExceptions;
 157   }
 158
 159   bool hasSDWA() const {
 160     return HasSDWA;
 161   }
 162
 163   bool hasVOP3PInsts() const {
 164     return HasVOP3PInsts;
 165   }
 166
 167   bool hasMulI24() const {
 168     return HasMulI24;
 169   }
 170
 171   bool hasMulU24() const {
 172     return HasMulU24;
 173   }
 174
 175   bool hasInv2PiInlineImm() const {
 176     return HasInv2PiInlineImm;
 177   }
 178
 179   bool hasFminFmaxLegacy() const {
 180     return HasFminFmaxLegacy;
 181   }
 182
 183   bool hasTrigReducedRange() const {
 184     return HasTrigReducedRange;
 185   }
 186
 187   bool isPromoteAllocaEnabled() const {
 188     return EnablePromoteAlloca;
 189   }
 190
 191   unsigned getWavefrontSize() const {
 192     return WavefrontSize;
 193   }
 194
 195   int getLocalMemorySize() const {
 196     return LocalMemorySize;
 197   }
 198
 199   unsigned getAlignmentForImplicitArgPtr() const {
 200     return isAmdHsaOS() ? 8 : 4;
 201   }
 202
 203   /// Returns the offset in bytes from the start of the input buffer
 204   ///        of the first explicit kernel argument.
 205   unsigned getExplicitKernelArgOffset(const Function &F) const {
 206     return isAmdHsaOrMesa(F) ? 0 : 36;
 207   }
 208
 209   /// \returns Maximum number of work groups per compute unit supported by the
 210   /// subtarget and limited by given \p FlatWorkGroupSize.
 211   virtual unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const = 0;
 212
 213   /// \returns Minimum flat work group size supported by the subtarget.
 214   virtual unsigned getMinFlatWorkGroupSize() const = 0;
 215
 216   /// \returns Maximum flat work group size supported by the subtarget.
 217   virtual unsigned getMaxFlatWorkGroupSize() const = 0;
 218
 219   /// \returns Maximum number of waves per execution unit supported by the
 220   /// subtarget and limited by given \p FlatWorkGroupSize.
 221   virtual unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const  = 0;
 222
 223   /// \returns Minimum number of waves per execution unit supported by the
 224   /// subtarget.
 225   virtual unsigned getMinWavesPerEU() const = 0;
 226
 227   /// \returns Maximum number of waves per execution unit supported by the
 228   /// subtarget without any kind of limitation.
 229   unsigned getMaxWavesPerEU() const { return MaxWavesPerEU; }
 230
 231   /// Creates value range metadata on an workitemid.* inrinsic call or load.
 232   bool makeLIDRangeMetadata(Instruction *I) const;
 233
 234   /// \returns Number of bytes of arguments that are passed to a shader or
 235   /// kernel in addition to the explicit ones declared for the function.
 236   unsigned getImplicitArgNumBytes(const Function &F) const {
 237     if (isMesaKernel(F))
 238       return 16;
 239     return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0);
 240   }
 241   uint64_t getExplicitKernArgSize(const Function &F,
 242                                   unsigned &MaxAlign) const;
 243   unsigned getKernArgSegmentSize(const Function &F,
 244                                  unsigned &MaxAlign) const;
 245
 246   virtual ~AMDGPUSubtarget() {}
 247 };
 248
 249 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
 250                      public AMDGPUSubtarget {
 251
 252   using AMDGPUSubtarget::getMaxWavesPerEU;
 253
 254 public:
 255   enum TrapHandlerAbi {
 256     TrapHandlerAbiNone = 0,
 257     TrapHandlerAbiHsa = 1
 258   };
 259
 260   enum TrapID {
 261     TrapIDHardwareReserved = 0,
 262     TrapIDHSADebugTrap = 1,
 263     TrapIDLLVMTrap = 2,
 264     TrapIDLLVMDebugTrap = 3,
 265     TrapIDDebugBreakpoint = 7,
 266     TrapIDDebugReserved8 = 8,
 267     TrapIDDebugReservedFE = 0xfe,
 268     TrapIDDebugReservedFF = 0xff
 269   };
 270
 271   enum TrapRegValues {
 272     LLVMTrapHandlerRegValue = 1
 273   };
 274
 275 private:
 276   /// GlobalISel related APIs.
 277   std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
 278   std::unique_ptr<InstructionSelector> InstSelector;
 279   std::unique_ptr<LegalizerInfo> Legalizer;
 280   std::unique_ptr<RegisterBankInfo> RegBankInfo;
 281
 282 protected:
 283   // Basic subtarget description.
 284   Triple TargetTriple;
 285   unsigned Gen;
 286   InstrItineraryData InstrItins;
 287   int LDSBankCount;
 288   unsigned MaxPrivateElementSize;
 289
 290   // Possibly statically set by tablegen, but may want to be overridden.
 291   bool FastFMAF32;
 292   bool HalfRate64Ops;
 293
 294   // Dynamially set bits that enable features.
 295   bool FP64FP16Denormals;
 296   bool FlatForGlobal;
 297   bool AutoWaitcntBeforeBarrier;
 298   bool CodeObjectV3;
 299   bool UnalignedScratchAccess;
 300   bool UnalignedBufferAccess;
 301   bool HasApertureRegs;
 302   bool EnableXNACK;
 303   bool DoesNotSupportXNACK;
 304   bool EnableCuMode;
 305   bool TrapHandler;
 306
 307   // Used as options.
 308   bool EnableLoadStoreOpt;
 309   bool EnableUnsafeDSOffsetFolding;
 310   bool EnableSIScheduler;
 311   bool EnableDS128;
 312   bool EnablePRTStrictNull;
 313   bool DumpCode;
 314
 315   // Subtarget statically properties set by tablegen
 316   bool FP64;
 317   bool FMA;
 318   bool MIMG_R128;
 319   bool IsGCN;
 320   bool GCN3Encoding;
 321   bool CIInsts;
 322   bool GFX8Insts;
 323   bool GFX9Insts;
 324   bool GFX10Insts;
 325   bool GFX7GFX8GFX9Insts;
 326   bool SGPRInitBug;
 327   bool HasSMemRealTime;
 328   bool HasIntClamp;
 329   bool HasFmaMixInsts;
 330   bool HasMovrel;
 331   bool HasVGPRIndexMode;
 332   bool HasScalarStores;
 333   bool HasScalarAtomics;
 334   bool HasSDWAOmod;
 335   bool HasSDWAScalar;
 336   bool HasSDWASdst;
 337   bool HasSDWAMac;
 338   bool HasSDWAOutModsVOPC;
 339   bool HasDPP;
 340   bool HasDPP8;
 341   bool HasR128A16;
 342   bool HasNSAEncoding;
 343   bool HasDLInsts;
 344   bool HasDot1Insts;
 345   bool HasDot2Insts;
 346   bool HasDot3Insts;
 347   bool HasDot4Insts;
 348   bool HasDot5Insts;
 349   bool HasDot6Insts;
 350   bool HasMAIInsts;
 351   bool HasPkFmacF16Inst;
 352   bool HasAtomicFaddInsts;
 353   bool EnableSRAMECC;
 354   bool DoesNotSupportSRAMECC;
 355   bool HasNoSdstCMPX;
 356   bool HasVscnt;
 357   bool HasRegisterBanking;
 358   bool HasVOP3Literal;
 359   bool HasNoDataDepHazard;
 360   bool FlatAddressSpace;
 361   bool FlatInstOffsets;
 362   bool FlatGlobalInsts;
 363   bool FlatScratchInsts;
 364   bool ScalarFlatScratchInsts;
 365   bool AddNoCarryInsts;
 366   bool HasUnpackedD16VMem;
 367   bool R600ALUInst;
 368   bool CaymanISA;
 369   bool CFALUBug;
 370   bool LDSMisalignedBug;
 371   bool HasMFMAInlineLiteralBug;
 372   bool HasVertexCache;
 373   short TexVTXClauseSize;
 374   bool ScalarizeGlobal;
 375
 376   bool HasVcmpxPermlaneHazard;
 377   bool HasVMEMtoScalarWriteHazard;
 378   bool HasSMEMtoVectorWriteHazard;
 379   bool HasInstFwdPrefetchBug;
 380   bool HasVcmpxExecWARHazard;
 381   bool HasLdsBranchVmemWARHazard;
 382   bool HasNSAtoVMEMBug;
 383   bool HasOffset3fBug;
 384   bool HasFlatSegmentOffsetBug;
 385
 386   // Dummy feature to use for assembler in tablegen.
 387   bool FeatureDisable;
 388
 389   SelectionDAGTargetInfo TSInfo;
 390 private:
 391   SIInstrInfo InstrInfo;
 392   SITargetLowering TLInfo;
 393   SIFrameLowering FrameLowering;
 394
 395   // See COMPUTE_TMPRING_SIZE.WAVESIZE, 13-bit field in units of 256-dword.
 396   static const unsigned MaxWaveScratchSize = (256 * 4) * ((1 << 13) - 1);
 397
 398 public:
 399   GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
 400                const GCNTargetMachine &TM);
 401   ~GCNSubtarget() override;
 402
 403   GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
 404                                                    StringRef GPU, StringRef FS);
 405
 406   const SIInstrInfo *getInstrInfo() const override {
 407     return &InstrInfo;
 408   }
 409
 410   const SIFrameLowering *getFrameLowering() const override {
 411     return &FrameLowering;
 412   }
 413
 414   const SITargetLowering *getTargetLowering() const override {
 415     return &TLInfo;
 416   }
 417
 418   const SIRegisterInfo *getRegisterInfo() const override {
 419     return &InstrInfo.getRegisterInfo();
 420   }
 421
 422   const CallLowering *getCallLowering() const override {
 423     return CallLoweringInfo.get();
 424   }
 425
 426   InstructionSelector *getInstructionSelector() const override {
 427     return InstSelector.get();
 428   }
 429
 430   const LegalizerInfo *getLegalizerInfo() const override {
 431     return Legalizer.get();
 432   }
 433
 434   const RegisterBankInfo *getRegBankInfo() const override {
 435     return RegBankInfo.get();
 436   }
 437
 438   // Nothing implemented, just prevent crashes on use.
 439   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
 440     return &TSInfo;
 441   }
 442
 443   const InstrItineraryData *getInstrItineraryData() const override {
 444     return &InstrItins;
 445   }
 446
 447   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
 448
 449   Generation getGeneration() const {
 450     return (Generation)Gen;
 451   }
 452
 453   unsigned getWavefrontSizeLog2() const {
 454     return Log2_32(WavefrontSize);
 455   }
 456
 457   /// Return the number of high bits known to be zero fror a frame index.
 458   unsigned getKnownHighZeroBitsForFrameIndex() const {
 459     return countLeadingZeros(MaxWaveScratchSize) + getWavefrontSizeLog2();
 460   }
 461
 462   int getLDSBankCount() const {
 463     return LDSBankCount;
 464   }
 465
 466   unsigned getMaxPrivateElementSize() const {
 467     return MaxPrivateElementSize;
 468   }
 469
 470   unsigned getConstantBusLimit(unsigned Opcode) const;
 471
 472   bool hasIntClamp() const {
 473     return HasIntClamp;
 474   }
 475
 476   bool hasFP64() const {
 477     return FP64;
 478   }
 479
 480   bool hasMIMG_R128() const {
 481     return MIMG_R128;
 482   }
 483
 484   bool hasHWFP64() const {
 485     return FP64;
 486   }
 487
 488   bool hasFastFMAF32() const {
 489     return FastFMAF32;
 490   }
 491
 492   bool hasHalfRate64Ops() const {
 493     return HalfRate64Ops;
 494   }
 495
 496   bool hasAddr64() const {
 497     return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);
 498   }
 499
 500   // Return true if the target only has the reverse operand versions of VALU
 501   // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).
 502   bool hasOnlyRevVALUShifts() const {
 503     return getGeneration() >= VOLCANIC_ISLANDS;
 504   }
 505
 506   bool hasBFE() const {
 507     return true;
 508   }
 509
 510   bool hasBFI() const {
 511     return true;
 512   }
 513
 514   bool hasBFM() const {
 515     return hasBFE();
 516   }
 517
 518   bool hasBCNT(unsigned Size) const {
 519     return true;
 520   }
 521
 522   bool hasFFBL() const {
 523     return true;
 524   }
 525
 526   bool hasFFBH() const {
 527     return true;
 528   }
 529
 530   bool hasMed3_16() const {
 531     return getGeneration() >= AMDGPUSubtarget::GFX9;
 532   }
 533
 534   bool hasMin3Max3_16() const {
 535     return getGeneration() >= AMDGPUSubtarget::GFX9;
 536   }
 537
 538   bool hasFmaMixInsts() const {
 539     return HasFmaMixInsts;
 540   }
 541
 542   bool hasCARRY() const {
 543     return true;
 544   }
 545
 546   bool hasFMA() const {
 547     return FMA;
 548   }
 549
 550   bool hasSwap() const {
 551     return GFX9Insts;
 552   }
 553
 554   bool hasScalarPackInsts() const {
 555     return GFX9Insts;
 556   }
 557
 558   bool hasScalarMulHiInsts() const {
 559     return GFX9Insts;
 560   }
 561
 562   TrapHandlerAbi getTrapHandlerAbi() const {
 563     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
 564   }
 565
 566   /// True if the offset field of DS instructions works as expected. On SI, the
 567   /// offset uses a 16-bit adder and does not always wrap properly.
 568   bool hasUsableDSOffset() const {
 569     return getGeneration() >= SEA_ISLANDS;
 570   }
 571
 572   bool unsafeDSOffsetFoldingEnabled() const {
 573     return EnableUnsafeDSOffsetFolding;
 574   }
 575
 576   /// Condition output from div_scale is usable.
 577   bool hasUsableDivScaleConditionOutput() const {
 578     return getGeneration() != SOUTHERN_ISLANDS;
 579   }
 580
 581   /// Extra wait hazard is needed in some cases before
 582   /// s_cbranch_vccnz/s_cbranch_vccz.
 583   bool hasReadVCCZBug() const {
 584     return getGeneration() <= SEA_ISLANDS;
 585   }
 586
 587   /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR
 588   /// was written by a VALU instruction.
 589   bool hasSMRDReadVALUDefHazard() const {
 590     return getGeneration() == SOUTHERN_ISLANDS;
 591   }
 592
 593   /// A read of an SGPR by a VMEM instruction requires 5 wait states when the
 594   /// SGPR was written by a VALU Instruction.
 595   bool hasVMEMReadSGPRVALUDefHazard() const {
 596     return getGeneration() >= VOLCANIC_ISLANDS;
 597   }
 598
 599   bool hasRFEHazards() const {
 600     return getGeneration() >= VOLCANIC_ISLANDS;
 601   }
 602
 603   /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.
 604   unsigned getSetRegWaitStates() const {
 605     return getGeneration() <= SEA_ISLANDS ? 1 : 2;
 606   }
 607
 608   bool dumpCode() const {
 609     return DumpCode;
 610   }
 611
 612   /// Return the amount of LDS that can be used that will not restrict the
 613   /// occupancy lower than WaveCount.
 614   unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,
 615                                            const Function &) const;
 616
 617   bool hasFP16Denormals() const {
 618     return FP64FP16Denormals;
 619   }
 620
 621   bool hasFP64Denormals() const {
 622     return FP64FP16Denormals;
 623   }
 624
 625   bool supportsMinMaxDenormModes() const {
 626     return getGeneration() >= AMDGPUSubtarget::GFX9;
 627   }
 628
 629   /// \returns If target supports S_DENORM_MODE.
 630   bool hasDenormModeInst() const {
 631     return getGeneration() >= AMDGPUSubtarget::GFX10;
 632   }
 633
 634   bool useFlatForGlobal() const {
 635     return FlatForGlobal;
 636   }
 637
 638   /// \returns If target supports ds_read/write_b128 and user enables generation
 639   /// of ds_read/write_b128.
 640   bool useDS128() const {
 641     return CIInsts && EnableDS128;
 642   }
 643
 644   /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64
 645   bool haveRoundOpsF64() const {
 646     return CIInsts;
 647   }
 648
 649   /// \returns If MUBUF instructions always perform range checking, even for
 650   /// buffer resources used for private memory access.
 651   bool privateMemoryResourceIsRangeChecked() const {
 652     return getGeneration() < AMDGPUSubtarget::GFX9;
 653   }
 654
 655   /// \returns If target requires PRT Struct NULL support (zero result registers
 656   /// for sparse texture support).
 657   bool usePRTStrictNull() const {
 658     return EnablePRTStrictNull;
 659   }
 660
 661   bool hasAutoWaitcntBeforeBarrier() const {
 662     return AutoWaitcntBeforeBarrier;
 663   }
 664
 665   bool hasCodeObjectV3() const {
 666     // FIXME: Need to add code object v3 support for mesa and pal.
 667     return isAmdHsaOS() ? CodeObjectV3 : false;
 668   }
 669
 670   bool hasUnalignedBufferAccess() const {
 671     return UnalignedBufferAccess;
 672   }
 673
 674   bool hasUnalignedScratchAccess() const {
 675     return UnalignedScratchAccess;
 676   }
 677
 678   bool hasApertureRegs() const {
 679     return HasApertureRegs;
 680   }
 681
 682   bool isTrapHandlerEnabled() const {
 683     return TrapHandler;
 684   }
 685
 686   bool isXNACKEnabled() const {
 687     return EnableXNACK;
 688   }
 689
 690   bool isCuModeEnabled() const {
 691     return EnableCuMode;
 692   }
 693
 694   bool hasFlatAddressSpace() const {
 695     return FlatAddressSpace;
 696   }
 697
 698   bool hasFlatScrRegister() const {
 699     return hasFlatAddressSpace();
 700   }
 701
 702   bool hasFlatInstOffsets() const {
 703     return FlatInstOffsets;
 704   }
 705
 706   bool hasFlatGlobalInsts() const {
 707     return FlatGlobalInsts;
 708   }
 709
 710   bool hasFlatScratchInsts() const {
 711     return FlatScratchInsts;
 712   }
 713
 714   bool hasScalarFlatScratchInsts() const {
 715     return ScalarFlatScratchInsts;
 716   }
 717
 718   bool hasFlatSegmentOffsetBug() const {
 719     return HasFlatSegmentOffsetBug;
 720   }
 721
 722   bool hasFlatLgkmVMemCountInOrder() const {
 723     return getGeneration() > GFX9;
 724   }
 725
 726   bool hasD16LoadStore() const {
 727     return getGeneration() >= GFX9;
 728   }
 729
 730   bool d16PreservesUnusedBits() const {
 731     return hasD16LoadStore() && !isSRAMECCEnabled();
 732   }
 733
 734   bool hasD16Images() const {
 735     return getGeneration() >= VOLCANIC_ISLANDS;
 736   }
 737
 738   /// Return if most LDS instructions have an m0 use that require m0 to be
 739   /// iniitalized.
 740   bool ldsRequiresM0Init() const {
 741     return getGeneration() < GFX9;
 742   }
 743
 744   // True if the hardware rewinds and replays GWS operations if a wave is
 745   // preempted.
 746   //
 747   // If this is false, a GWS operation requires testing if a nack set the
 748   // MEM_VIOL bit, and repeating if so.
 749   bool hasGWSAutoReplay() const {
 750     return getGeneration() >= GFX9;
 751   }
 752
 753   /// \returns if target has ds_gws_sema_release_all instruction.
 754   bool hasGWSSemaReleaseAll() const {
 755     return CIInsts;
 756   }
 757
 758   bool hasAddNoCarry() const {
 759     return AddNoCarryInsts;
 760   }
 761
 762   bool hasUnpackedD16VMem() const {
 763     return HasUnpackedD16VMem;
 764   }
 765
 766   // Covers VS/PS/CS graphics shaders
 767   bool isMesaGfxShader(const Function &F) const {
 768     return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());
 769   }
 770
 771   bool hasMad64_32() const {
 772     return getGeneration() >= SEA_ISLANDS;
 773   }
 774
 775   bool hasSDWAOmod() const {
 776     return HasSDWAOmod;
 777   }
 778
 779   bool hasSDWAScalar() const {
 780     return HasSDWAScalar;
 781   }
 782
 783   bool hasSDWASdst() const {
 784     return HasSDWASdst;
 785   }
 786
 787   bool hasSDWAMac() const {
 788     return HasSDWAMac;
 789   }
 790
 791   bool hasSDWAOutModsVOPC() const {
 792     return HasSDWAOutModsVOPC;
 793   }
 794
 795   bool hasDLInsts() const {
 796     return HasDLInsts;
 797   }
 798
 799   bool hasDot1Insts() const {
 800     return HasDot1Insts;
 801   }
 802
 803   bool hasDot2Insts() const {
 804     return HasDot2Insts;
 805   }
 806
 807   bool hasDot3Insts() const {
 808     return HasDot3Insts;
 809   }
 810
 811   bool hasDot4Insts() const {
 812     return HasDot4Insts;
 813   }
 814
 815   bool hasDot5Insts() const {
 816     return HasDot5Insts;
 817   }
 818
 819   bool hasDot6Insts() const {
 820     return HasDot6Insts;
 821   }
 822
 823   bool hasMAIInsts() const {
 824     return HasMAIInsts;
 825   }
 826
 827   bool hasPkFmacF16Inst() const {
 828     return HasPkFmacF16Inst;
 829   }
 830
 831   bool hasAtomicFaddInsts() const {
 832     return HasAtomicFaddInsts;
 833   }
 834
 835   bool isSRAMECCEnabled() const {
 836     return EnableSRAMECC;
 837   }
 838
 839   bool hasNoSdstCMPX() const {
 840     return HasNoSdstCMPX;
 841   }
 842
 843   bool hasVscnt() const {
 844     return HasVscnt;
 845   }
 846
 847   bool hasRegisterBanking() const {
 848     return HasRegisterBanking;
 849   }
 850
 851   bool hasVOP3Literal() const {
 852     return HasVOP3Literal;
 853   }
 854
 855   bool hasNoDataDepHazard() const {
 856     return HasNoDataDepHazard;
 857   }
 858
 859   bool vmemWriteNeedsExpWaitcnt() const {
 860     return getGeneration() < SEA_ISLANDS;
 861   }
 862
 863   // Scratch is allocated in 256 dword per wave blocks for the entire
 864   // wavefront. When viewed from the perspecive of an arbitrary workitem, this
 865   // is 4-byte aligned.
 866   //
 867   // Only 4-byte alignment is really needed to access anything. Transformations
 868   // on the pointer value itself may rely on the alignment / known low bits of
 869   // the pointer. Set this to something above the minimum to avoid needing
 870   // dynamic realignment in common cases.
 871   unsigned getStackAlignment() const {
 872     return 16;
 873   }
 874
 875   bool enableMachineScheduler() const override {
 876     return true;
 877   }
 878
 879   bool enableSubRegLiveness() const override {
 880     return true;
 881   }
 882
 883   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
 884   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
 885
 886   /// \returns Number of execution units per compute unit supported by the
 887   /// subtarget.
 888   unsigned getEUsPerCU() const {
 889     return AMDGPU::IsaInfo::getEUsPerCU(this);
 890   }
 891
 892   /// \returns Maximum number of waves per compute unit supported by the
 893   /// subtarget without any kind of limitation.
 894   unsigned getMaxWavesPerCU() const {
 895     return AMDGPU::IsaInfo::getMaxWavesPerCU(this);
 896   }
 897
 898   /// \returns Maximum number of waves per compute unit supported by the
 899   /// subtarget and limited by given \p FlatWorkGroupSize.
 900   unsigned getMaxWavesPerCU(unsigned FlatWorkGroupSize) const {
 901     return AMDGPU::IsaInfo::getMaxWavesPerCU(this, FlatWorkGroupSize);
 902   }
 903
 904   /// \returns Number of waves per work group supported by the subtarget and
 905   /// limited by given \p FlatWorkGroupSize.
 906   unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
 907     return AMDGPU::IsaInfo::getWavesPerWorkGroup(this, FlatWorkGroupSize);
 908   }
 909
 910   // static wrappers
 911   static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
 912
 913   // XXX - Why is this here if it isn't in the default pass set?
 914   bool enableEarlyIfConversion() const override {
 915     return true;
 916   }
 917
 918   void overrideSchedPolicy(MachineSchedPolicy &Policy,
 919                            unsigned NumRegionInstrs) const override;
 920
 921   unsigned getMaxNumUserSGPRs() const {
 922     return 16;
 923   }
 924
 925   bool hasSMemRealTime() const {
 926     return HasSMemRealTime;
 927   }
 928
 929   bool hasMovrel() const {
 930     return HasMovrel;
 931   }
 932
 933   bool hasVGPRIndexMode() const {
 934     return HasVGPRIndexMode;
 935   }
 936
 937   bool useVGPRIndexMode(bool UserEnable) const {
 938     return !hasMovrel() || (UserEnable && hasVGPRIndexMode());
 939   }
 940
 941   bool hasScalarCompareEq64() const {
 942     return getGeneration() >= VOLCANIC_ISLANDS;
 943   }
 944
 945   bool hasScalarStores() const {
 946     return HasScalarStores;
 947   }
 948
 949   bool hasScalarAtomics() const {
 950     return HasScalarAtomics;
 951   }
 952
 953   bool hasLDSFPAtomics() const {
 954     return GFX8Insts;
 955   }
 956
 957   bool hasDPP() const {
 958     return HasDPP;
 959   }
 960
 961   bool hasDPPBroadcasts() const {
 962     return HasDPP && getGeneration() < GFX10;
 963   }
 964
 965   bool hasDPPWavefrontShifts() const {
 966     return HasDPP && getGeneration() < GFX10;
 967   }
 968
 969   bool hasDPP8() const {
 970     return HasDPP8;
 971   }
 972
 973   bool hasR128A16() const {
 974     return HasR128A16;
 975   }
 976
 977   bool hasOffset3fBug() const {
 978     return HasOffset3fBug;
 979   }
 980
 981   bool hasNSAEncoding() const {
 982     return HasNSAEncoding;
 983   }
 984
 985   bool hasMadF16() const;
 986
 987   bool enableSIScheduler() const {
 988     return EnableSIScheduler;
 989   }
 990
 991   bool loadStoreOptEnabled() const {
 992     return EnableLoadStoreOpt;
 993   }
 994
 995   bool hasSGPRInitBug() const {
 996     return SGPRInitBug;
 997   }
 998
 999   bool hasMFMAInlineLiteralBug() const {
1000     return HasMFMAInlineLiteralBug;
1001   }
1002
1003   bool has12DWordStoreHazard() const {
1004     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
1005   }
1006
1007   // \returns true if the subtarget supports DWORDX3 load/store instructions.
1008   bool hasDwordx3LoadStores() const {
1009     return CIInsts;
1010   }
1011
1012   bool hasSMovFedHazard() const {
1013     return getGeneration() == AMDGPUSubtarget::GFX9;
1014   }
1015
1016   bool hasReadM0MovRelInterpHazard() const {
1017     return getGeneration() == AMDGPUSubtarget::GFX9;
1018   }
1019
1020   bool hasReadM0SendMsgHazard() const {
1021     return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1022            getGeneration() <= AMDGPUSubtarget::GFX9;
1023   }
1024
1025   bool hasVcmpxPermlaneHazard() const {
1026     return HasVcmpxPermlaneHazard;
1027   }
1028
1029   bool hasVMEMtoScalarWriteHazard() const {
1030     return HasVMEMtoScalarWriteHazard;
1031   }
1032
1033   bool hasSMEMtoVectorWriteHazard() const {
1034     return HasSMEMtoVectorWriteHazard;
1035   }
1036
1037   bool hasLDSMisalignedBug() const {
1038     return LDSMisalignedBug && !EnableCuMode;
1039   }
1040
1041   bool hasInstFwdPrefetchBug() const {
1042     return HasInstFwdPrefetchBug;
1043   }
1044
1045   bool hasVcmpxExecWARHazard() const {
1046     return HasVcmpxExecWARHazard;
1047   }
1048
1049   bool hasLdsBranchVmemWARHazard() const {
1050     return HasLdsBranchVmemWARHazard;
1051   }
1052
1053   bool hasNSAtoVMEMBug() const {
1054     return HasNSAtoVMEMBug;
1055   }
1056
1057   /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
1058   /// SGPRs
1059   unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
1060
1061   /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
1062   /// VGPRs
1063   unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
1064
1065   /// Return occupancy for the given function. Used LDS and a number of
1066   /// registers if provided.
1067   /// Note, occupancy can be affected by the scratch allocation as well, but
1068   /// we do not have enough information to compute it.
1069   unsigned computeOccupancy(const MachineFunction &MF, unsigned LDSSize = 0,
1070                             unsigned NumSGPRs = 0, unsigned NumVGPRs = 0) const;
1071
1072   /// \returns true if the flat_scratch register should be initialized with the
1073   /// pointer to the wave's scratch memory rather than a size and offset.
1074   bool flatScratchIsPointer() const {
1075     return getGeneration() >= AMDGPUSubtarget::GFX9;
1076   }
1077
1078   /// \returns true if the machine has merged shaders in which s0-s7 are
1079   /// reserved by the hardware and user SGPRs start at s8
1080   bool hasMergedShaders() const {
1081     return getGeneration() >= GFX9;
1082   }
1083
1084   /// \returns SGPR allocation granularity supported by the subtarget.
1085   unsigned getSGPRAllocGranule() const {
1086     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
1087   }
1088
1089   /// \returns SGPR encoding granularity supported by the subtarget.
1090   unsigned getSGPREncodingGranule() const {
1091     return AMDGPU::IsaInfo::getSGPREncodingGranule(this);
1092   }
1093
1094   /// \returns Total number of SGPRs supported by the subtarget.
1095   unsigned getTotalNumSGPRs() const {
1096     return AMDGPU::IsaInfo::getTotalNumSGPRs(this);
1097   }
1098
1099   /// \returns Addressable number of SGPRs supported by the subtarget.
1100   unsigned getAddressableNumSGPRs() const {
1101     return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);
1102   }
1103
1104   /// \returns Minimum number of SGPRs that meets the given number of waves per
1105   /// execution unit requirement supported by the subtarget.
1106   unsigned getMinNumSGPRs(unsigned WavesPerEU) const {
1107     return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);
1108   }
1109
1110   /// \returns Maximum number of SGPRs that meets the given number of waves per
1111   /// execution unit requirement supported by the subtarget.
1112   unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {
1113     return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);
1114   }
1115
1116   /// \returns Reserved number of SGPRs for given function \p MF.
1117   unsigned getReservedNumSGPRs(const MachineFunction &MF) const;
1118
1119   /// \returns Maximum number of SGPRs that meets number of waves per execution
1120   /// unit requirement for function \p MF, or number of SGPRs explicitly
1121   /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.
1122   ///
1123   /// \returns Value that meets number of waves per execution unit requirement
1124   /// if explicitly requested value cannot be converted to integer, violates
1125   /// subtarget's specifications, or does not meet number of waves per execution
1126   /// unit requirement.
1127   unsigned getMaxNumSGPRs(const MachineFunction &MF) const;
1128
1129   /// \returns VGPR allocation granularity supported by the subtarget.
1130   unsigned getVGPRAllocGranule() const {
1131     return AMDGPU::IsaInfo::getVGPRAllocGranule(this);
1132   }
1133
1134   /// \returns VGPR encoding granularity supported by the subtarget.
1135   unsigned getVGPREncodingGranule() const {
1136     return AMDGPU::IsaInfo::getVGPREncodingGranule(this);
1137   }
1138
1139   /// \returns Total number of VGPRs supported by the subtarget.
1140   unsigned getTotalNumVGPRs() const {
1141     return AMDGPU::IsaInfo::getTotalNumVGPRs(this);
1142   }
1143
1144   /// \returns Addressable number of VGPRs supported by the subtarget.
1145   unsigned getAddressableNumVGPRs() const {
1146     return AMDGPU::IsaInfo::getAddressableNumVGPRs(this);
1147   }
1148
1149   /// \returns Minimum number of VGPRs that meets given number of waves per
1150   /// execution unit requirement supported by the subtarget.
1151   unsigned getMinNumVGPRs(unsigned WavesPerEU) const {
1152     return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU);
1153   }
1154
1155   /// \returns Maximum number of VGPRs that meets given number of waves per
1156   /// execution unit requirement supported by the subtarget.
1157   unsigned getMaxNumVGPRs(unsigned WavesPerEU) const {
1158     return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU);
1159   }
1160
1161   /// \returns Maximum number of VGPRs that meets number of waves per execution
1162   /// unit requirement for function \p MF, or number of VGPRs explicitly
1163   /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.
1164   ///
1165   /// \returns Value that meets number of waves per execution unit requirement
1166   /// if explicitly requested value cannot be converted to integer, violates
1167   /// subtarget's specifications, or does not meet number of waves per execution
1168   /// unit requirement.
1169   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
1170
1171   void getPostRAMutations(
1172       std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
1173       const override;
1174
1175   bool isWave32() const {
1176     return WavefrontSize == 32;
1177   }
1178
1179   const TargetRegisterClass *getBoolRC() const {
1180     return getRegisterInfo()->getBoolRC();
1181   }
1182
1183   /// \returns Maximum number of work groups per compute unit supported by the
1184   /// subtarget and limited by given \p FlatWorkGroupSize.
1185   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1186     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1187   }
1188
1189   /// \returns Minimum flat work group size supported by the subtarget.
1190   unsigned getMinFlatWorkGroupSize() const override {
1191     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1192   }
1193
1194   /// \returns Maximum flat work group size supported by the subtarget.
1195   unsigned getMaxFlatWorkGroupSize() const override {
1196     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1197   }
1198
1199   /// \returns Maximum number of waves per execution unit supported by the
1200   /// subtarget and limited by given \p FlatWorkGroupSize.
1201   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1202     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1203   }
1204
1205   /// \returns Minimum number of waves per execution unit supported by the
1206   /// subtarget.
1207   unsigned getMinWavesPerEU() const override {
1208     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1209   }
1210 };
1211
1212 class R600Subtarget final : public R600GenSubtargetInfo,
1213                             public AMDGPUSubtarget {
1214 private:
1215   R600InstrInfo InstrInfo;
1216   R600FrameLowering FrameLowering;
1217   bool FMA;
1218   bool CaymanISA;
1219   bool CFALUBug;
1220   bool HasVertexCache;
1221   bool R600ALUInst;
1222   bool FP64;
1223   short TexVTXClauseSize;
1224   Generation Gen;
1225   R600TargetLowering TLInfo;
1226   InstrItineraryData InstrItins;
1227   SelectionDAGTargetInfo TSInfo;
1228
1229 public:
1230   R600Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
1231                 const TargetMachine &TM);
1232
1233   const R600InstrInfo *getInstrInfo() const override { return &InstrInfo; }
1234
1235   const R600FrameLowering *getFrameLowering() const override {
1236     return &FrameLowering;
1237   }
1238
1239   const R600TargetLowering *getTargetLowering() const override {
1240     return &TLInfo;
1241   }
1242
1243   const R600RegisterInfo *getRegisterInfo() const override {
1244     return &InstrInfo.getRegisterInfo();
1245   }
1246
1247   const InstrItineraryData *getInstrItineraryData() const override {
1248     return &InstrItins;
1249   }
1250
1251   // Nothing implemented, just prevent crashes on use.
1252   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
1253     return &TSInfo;
1254   }
1255
1256   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
1257
1258   Generation getGeneration() const {
1259     return Gen;
1260   }
1261
1262   unsigned getStackAlignment() const {
1263     return 4;
1264   }
1265
1266   R600Subtarget &initializeSubtargetDependencies(const Triple &TT,
1267                                                  StringRef GPU, StringRef FS);
1268
1269   bool hasBFE() const {
1270     return (getGeneration() >= EVERGREEN);
1271   }
1272
1273   bool hasBFI() const {
1274     return (getGeneration() >= EVERGREEN);
1275   }
1276
1277   bool hasBCNT(unsigned Size) const {
1278     if (Size == 32)
1279       return (getGeneration() >= EVERGREEN);
1280
1281     return false;
1282   }
1283
1284   bool hasBORROW() const {
1285     return (getGeneration() >= EVERGREEN);
1286   }
1287
1288   bool hasCARRY() const {
1289     return (getGeneration() >= EVERGREEN);
1290   }
1291
1292   bool hasCaymanISA() const {
1293     return CaymanISA;
1294   }
1295
1296   bool hasFFBL() const {
1297     return (getGeneration() >= EVERGREEN);
1298   }
1299
1300   bool hasFFBH() const {
1301     return (getGeneration() >= EVERGREEN);
1302   }
1303
1304   bool hasFMA() const { return FMA; }
1305
1306   bool hasCFAluBug() const { return CFALUBug; }
1307
1308   bool hasVertexCache() const { return HasVertexCache; }
1309
1310   short getTexVTXClauseSize() const { return TexVTXClauseSize; }
1311
1312   bool enableMachineScheduler() const override {
1313     return true;
1314   }
1315
1316   bool enableSubRegLiveness() const override {
1317     return true;
1318   }
1319
1320   /// \returns Maximum number of work groups per compute unit supported by the
1321   /// subtarget and limited by given \p FlatWorkGroupSize.
1322   unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {
1323     return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);
1324   }
1325
1326   /// \returns Minimum flat work group size supported by the subtarget.
1327   unsigned getMinFlatWorkGroupSize() const override {
1328     return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);
1329   }
1330
1331   /// \returns Maximum flat work group size supported by the subtarget.
1332   unsigned getMaxFlatWorkGroupSize() const override {
1333     return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);
1334   }
1335
1336   /// \returns Maximum number of waves per execution unit supported by the
1337   /// subtarget and limited by given \p FlatWorkGroupSize.
1338   unsigned getMaxWavesPerEU(unsigned FlatWorkGroupSize) const override {
1339     return AMDGPU::IsaInfo::getMaxWavesPerEU(this, FlatWorkGroupSize);
1340   }
1341
1342   /// \returns Minimum number of waves per execution unit supported by the
1343   /// subtarget.
1344   unsigned getMinWavesPerEU() const override {
1345     return AMDGPU::IsaInfo::getMinWavesPerEU(this);
1346   }
1347 };
1348
1349 } // end namespace llvm
1350
1351 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H