lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

   1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "AMDGPUBaseInfo.h"
  10 #include "AMDGPUTargetTransformInfo.h"
  11 #include "AMDGPU.h"
  12 #include "SIDefines.h"
  13 #include "AMDGPUAsmUtils.h"
  14 #include "llvm/ADT/StringRef.h"
  15 #include "llvm/ADT/Triple.h"
  16 #include "llvm/BinaryFormat/ELF.h"
  17 #include "llvm/CodeGen/MachineMemOperand.h"
  18 #include "llvm/IR/Attributes.h"
  19 #include "llvm/IR/Constants.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/IR/GlobalValue.h"
  22 #include "llvm/IR/Instruction.h"
  23 #include "llvm/IR/LLVMContext.h"
  24 #include "llvm/IR/Module.h"
  25 #include "llvm/MC/MCContext.h"
  26 #include "llvm/MC/MCInstrDesc.h"
  27 #include "llvm/MC/MCInstrInfo.h"
  28 #include "llvm/MC/MCRegisterInfo.h"
  29 #include "llvm/MC/MCSectionELF.h"
  30 #include "llvm/MC/MCSubtargetInfo.h"
  31 #include "llvm/MC/SubtargetFeature.h"
  32 #include "llvm/Support/Casting.h"
  33 #include "llvm/Support/ErrorHandling.h"
  34 #include "llvm/Support/MathExtras.h"
  35 #include <algorithm>
  36 #include <cassert>
  37 #include <cstdint>
  38 #include <cstring>
  39 #include <utility>
  40
  41 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  42
  43 #define GET_INSTRINFO_NAMED_OPS
  44 #define GET_INSTRMAP_INFO
  45 #include "AMDGPUGenInstrInfo.inc"
  46 #undef GET_INSTRMAP_INFO
  47 #undef GET_INSTRINFO_NAMED_OPS
  48
  49 namespace {
  50
  51 /// \returns Bit mask for given bit \p Shift and bit \p Width.
  52 unsigned getBitMask(unsigned Shift, unsigned Width) {
  53   return ((1 << Width) - 1) << Shift;
  54 }
  55
  56 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
  57 ///
  58 /// \returns Packed \p Dst.
  59 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
  60   Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width);
  61   Dst |= (Src << Shift) & getBitMask(Shift, Width);
  62   return Dst;
  63 }
  64
  65 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
  66 ///
  67 /// \returns Unpacked bits.
  68 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
  69   return (Src & getBitMask(Shift, Width)) >> Shift;
  70 }
  71
  72 /// \returns Vmcnt bit shift (lower bits).
  73 unsigned getVmcntBitShiftLo() { return 0; }
  74
  75 /// \returns Vmcnt bit width (lower bits).
  76 unsigned getVmcntBitWidthLo() { return 4; }
  77
  78 /// \returns Expcnt bit shift.
  79 unsigned getExpcntBitShift() { return 4; }
  80
  81 /// \returns Expcnt bit width.
  82 unsigned getExpcntBitWidth() { return 3; }
  83
  84 /// \returns Lgkmcnt bit shift.
  85 unsigned getLgkmcntBitShift() { return 8; }
  86
  87 /// \returns Lgkmcnt bit width.
  88 unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
  89   return (VersionMajor >= 10) ? 6 : 4;
  90 }
  91
  92 /// \returns Vmcnt bit shift (higher bits).
  93 unsigned getVmcntBitShiftHi() { return 14; }
  94
  95 /// \returns Vmcnt bit width (higher bits).
  96 unsigned getVmcntBitWidthHi() { return 2; }
  97
  98 } // end namespace anonymous
  99
 100 namespace llvm {
 101
 102 namespace AMDGPU {
 103
 104 #define GET_MIMGBaseOpcodesTable_IMPL
 105 #define GET_MIMGDimInfoTable_IMPL
 106 #define GET_MIMGInfoTable_IMPL
 107 #define GET_MIMGLZMappingTable_IMPL
 108 #define GET_MIMGMIPMappingTable_IMPL
 109 #include "AMDGPUGenSearchableTables.inc"
 110
 111 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
 112                   unsigned VDataDwords, unsigned VAddrDwords) {
 113   const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding,
 114                                              VDataDwords, VAddrDwords);
 115   return Info ? Info->Opcode : -1;
 116 }
 117
 118 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
 119   const MIMGInfo *Info = getMIMGInfo(Opc);
 120   return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
 121 }
 122
 123 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
 124   const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
 125   const MIMGInfo *NewInfo =
 126       getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding,
 127                           NewChannels, OrigInfo->VAddrDwords);
 128   return NewInfo ? NewInfo->Opcode : -1;
 129 }
 130
 131 struct MUBUFInfo {
 132   uint16_t Opcode;
 133   uint16_t BaseOpcode;
 134   uint8_t elements;
 135   bool has_vaddr;
 136   bool has_srsrc;
 137   bool has_soffset;
 138 };
 139
 140 #define GET_MUBUFInfoTable_DECL
 141 #define GET_MUBUFInfoTable_IMPL
 142 #include "AMDGPUGenSearchableTables.inc"
 143
 144 int getMUBUFBaseOpcode(unsigned Opc) {
 145   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
 146   return Info ? Info->BaseOpcode : -1;
 147 }
 148
 149 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
 150   const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
 151   return Info ? Info->Opcode : -1;
 152 }
 153
 154 int getMUBUFElements(unsigned Opc) {
 155   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
 156   return Info ? Info->elements : 0;
 157 }
 158
 159 bool getMUBUFHasVAddr(unsigned Opc) {
 160   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
 161   return Info ? Info->has_vaddr : false;
 162 }
 163
 164 bool getMUBUFHasSrsrc(unsigned Opc) {
 165   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
 166   return Info ? Info->has_srsrc : false;
 167 }
 168
 169 bool getMUBUFHasSoffset(unsigned Opc) {
 170   const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
 171   return Info ? Info->has_soffset : false;
 172 }
 173
 174 // Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
 175 // header files, so we need to wrap it in a function that takes unsigned
 176 // instead.
 177 int getMCOpcode(uint16_t Opcode, unsigned Gen) {
 178   return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
 179 }
 180
 181 namespace IsaInfo {
 182
 183 void streamIsaVersion(const MCSubtargetInfo *STI, raw_ostream &Stream) {
 184   auto TargetTriple = STI->getTargetTriple();
 185   auto Version = getIsaVersion(STI->getCPU());
 186
 187   Stream << TargetTriple.getArchName() << '-'
 188          << TargetTriple.getVendorName() << '-'
 189          << TargetTriple.getOSName() << '-'
 190          << TargetTriple.getEnvironmentName() << '-'
 191          << "gfx"
 192          << Version.Major
 193          << Version.Minor
 194          << Version.Stepping;
 195
 196   if (hasXNACK(*STI))
 197     Stream << "+xnack";
 198   if (hasSRAMECC(*STI))
 199     Stream << "+sram-ecc";
 200
 201   Stream.flush();
 202 }
 203
 204 bool hasCodeObjectV3(const MCSubtargetInfo *STI) {
 205   return STI->getTargetTriple().getOS() == Triple::AMDHSA &&
 206              STI->getFeatureBits().test(FeatureCodeObjectV3);
 207 }
 208
 209 unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
 210   if (STI->getFeatureBits().test(FeatureWavefrontSize16))
 211     return 16;
 212   if (STI->getFeatureBits().test(FeatureWavefrontSize32))
 213     return 32;
 214
 215   return 64;
 216 }
 217
 218 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
 219   if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
 220     return 32768;
 221   if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
 222     return 65536;
 223
 224   return 0;
 225 }
 226
 227 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
 228   return 4;
 229 }
 230
 231 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
 232                                unsigned FlatWorkGroupSize) {
 233   assert(FlatWorkGroupSize != 0);
 234   if (STI->getTargetTriple().getArch() != Triple::amdgcn)
 235     return 8;
 236   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
 237   if (N == 1)
 238     return 40;
 239   N = 40 / N;
 240   return std::min(N, 16u);
 241 }
 242
 243 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI) {
 244   return getMaxWavesPerEU(STI) * getEUsPerCU(STI);
 245 }
 246
 247 unsigned getMaxWavesPerCU(const MCSubtargetInfo *STI,
 248                           unsigned FlatWorkGroupSize) {
 249   return getWavesPerWorkGroup(STI, FlatWorkGroupSize);
 250 }
 251
 252 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) {
 253   return 1;
 254 }
 255
 256 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) {
 257   // FIXME: Need to take scratch memory into account.
 258   if (!isGFX10(*STI))
 259     return 10;
 260   return 20;
 261 }
 262
 263 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI,
 264                           unsigned FlatWorkGroupSize) {
 265   return alignTo(getMaxWavesPerCU(STI, FlatWorkGroupSize),
 266                  getEUsPerCU(STI)) / getEUsPerCU(STI);
 267 }
 268
 269 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) {
 270   return 1;
 271 }
 272
 273 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) {
 274   return 2048;
 275 }
 276
 277 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI,
 278                               unsigned FlatWorkGroupSize) {
 279   return alignTo(FlatWorkGroupSize, getWavefrontSize(STI)) /
 280                  getWavefrontSize(STI);
 281 }
 282
 283 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) {
 284   IsaVersion Version = getIsaVersion(STI->getCPU());
 285   if (Version.Major >= 10)
 286     return getAddressableNumSGPRs(STI);
 287   if (Version.Major >= 8)
 288     return 16;
 289   return 8;
 290 }
 291
 292 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) {
 293   return 8;
 294 }
 295
 296 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) {
 297   IsaVersion Version = getIsaVersion(STI->getCPU());
 298   if (Version.Major >= 8)
 299     return 800;
 300   return 512;
 301 }
 302
 303 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) {
 304   if (STI->getFeatureBits().test(FeatureSGPRInitBug))
 305     return FIXED_NUM_SGPRS_FOR_INIT_BUG;
 306
 307   IsaVersion Version = getIsaVersion(STI->getCPU());
 308   if (Version.Major >= 10)
 309     return 106;
 310   if (Version.Major >= 8)
 311     return 102;
 312   return 104;
 313 }
 314
 315 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
 316   assert(WavesPerEU != 0);
 317
 318   IsaVersion Version = getIsaVersion(STI->getCPU());
 319   if (Version.Major >= 10)
 320     return 0;
 321
 322   if (WavesPerEU >= getMaxWavesPerEU(STI))
 323     return 0;
 324
 325   unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1);
 326   if (STI->getFeatureBits().test(FeatureTrapHandler))
 327     MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
 328   MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1;
 329   return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI));
 330 }
 331
 332 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU,
 333                         bool Addressable) {
 334   assert(WavesPerEU != 0);
 335
 336   unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
 337   IsaVersion Version = getIsaVersion(STI->getCPU());
 338   if (Version.Major >= 10)
 339     return Addressable ? AddressableNumSGPRs : 108;
 340   if (Version.Major >= 8 && !Addressable)
 341     AddressableNumSGPRs = 112;
 342   unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU;
 343   if (STI->getFeatureBits().test(FeatureTrapHandler))
 344     MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS);
 345   MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI));
 346   return std::min(MaxNumSGPRs, AddressableNumSGPRs);
 347 }
 348
 349 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
 350                           bool FlatScrUsed, bool XNACKUsed) {
 351   unsigned ExtraSGPRs = 0;
 352   if (VCCUsed)
 353     ExtraSGPRs = 2;
 354
 355   IsaVersion Version = getIsaVersion(STI->getCPU());
 356   if (Version.Major >= 10)
 357     return ExtraSGPRs;
 358
 359   if (Version.Major < 8) {
 360     if (FlatScrUsed)
 361       ExtraSGPRs = 4;
 362   } else {
 363     if (XNACKUsed)
 364       ExtraSGPRs = 4;
 365
 366     if (FlatScrUsed)
 367       ExtraSGPRs = 6;
 368   }
 369
 370   return ExtraSGPRs;
 371 }
 372
 373 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed,
 374                           bool FlatScrUsed) {
 375   return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
 376                           STI->getFeatureBits().test(AMDGPU::FeatureXNACK));
 377 }
 378
 379 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) {
 380   NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI));
 381   // SGPRBlocks is actual number of SGPR blocks minus 1.
 382   return NumSGPRs / getSGPREncodingGranule(STI) - 1;
 383 }
 384
 385 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI,
 386                              Optional<bool> EnableWavefrontSize32) {
 387   bool IsWave32 = EnableWavefrontSize32 ?
 388       *EnableWavefrontSize32 :
 389       STI->getFeatureBits().test(FeatureWavefrontSize32);
 390   return IsWave32 ? 8 : 4;
 391 }
 392
 393 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI,
 394                                 Optional<bool> EnableWavefrontSize32) {
 395   return getVGPRAllocGranule(STI, EnableWavefrontSize32);
 396 }
 397
 398 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) {
 399   if (!isGFX10(*STI))
 400     return 256;
 401   return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512;
 402 }
 403
 404 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) {
 405   return 256;
 406 }
 407
 408 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
 409   assert(WavesPerEU != 0);
 410
 411   if (WavesPerEU >= getMaxWavesPerEU(STI))
 412     return 0;
 413   unsigned MinNumVGPRs =
 414       alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1),
 415                 getVGPRAllocGranule(STI)) + 1;
 416   return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI));
 417 }
 418
 419 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) {
 420   assert(WavesPerEU != 0);
 421
 422   unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU,
 423                                    getVGPRAllocGranule(STI));
 424   unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI);
 425   return std::min(MaxNumVGPRs, AddressableNumVGPRs);
 426 }
 427
 428 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs,
 429                           Optional<bool> EnableWavefrontSize32) {
 430   NumVGPRs = alignTo(std::max(1u, NumVGPRs),
 431                      getVGPREncodingGranule(STI, EnableWavefrontSize32));
 432   // VGPRBlocks is actual number of VGPR blocks minus 1.
 433   return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1;
 434 }
 435
 436 } // end namespace IsaInfo
 437
 438 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header,
 439                                const MCSubtargetInfo *STI) {
 440   IsaVersion Version = getIsaVersion(STI->getCPU());
 441
 442   memset(&Header, 0, sizeof(Header));
 443
 444   Header.amd_kernel_code_version_major = 1;
 445   Header.amd_kernel_code_version_minor = 2;
 446   Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
 447   Header.amd_machine_version_major = Version.Major;
 448   Header.amd_machine_version_minor = Version.Minor;
 449   Header.amd_machine_version_stepping = Version.Stepping;
 450   Header.kernel_code_entry_byte_offset = sizeof(Header);
 451   Header.wavefront_size = 6;
 452
 453   // If the code object does not support indirect functions, then the value must
 454   // be 0xffffffff.
 455   Header.call_convention = -1;
 456
 457   // These alignment values are specified in powers of two, so alignment =
 458   // 2^n.  The minimum alignment is 2^4 = 16.
 459   Header.kernarg_segment_alignment = 4;
 460   Header.group_segment_alignment = 4;
 461   Header.private_segment_alignment = 4;
 462
 463   if (Version.Major >= 10) {
 464     if (STI->getFeatureBits().test(FeatureWavefrontSize32)) {
 465       Header.wavefront_size = 5;
 466       Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
 467     }
 468     Header.compute_pgm_resource_registers |=
 469       S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
 470       S_00B848_MEM_ORDERED(1);
 471   }
 472 }
 473
 474 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
 475     const MCSubtargetInfo *STI) {
 476   IsaVersion Version = getIsaVersion(STI->getCPU());
 477
 478   amdhsa::kernel_descriptor_t KD;
 479   memset(&KD, 0, sizeof(KD));
 480
 481   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
 482                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
 483                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
 484   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
 485                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
 486   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
 487                   amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
 488   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
 489                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
 490   if (Version.Major >= 10) {
 491     AMDHSA_BITS_SET(KD.kernel_code_properties,
 492                     amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
 493                     STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0);
 494     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
 495                     amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE,
 496                     STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1);
 497     AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
 498                     amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1);
 499   }
 500   return KD;
 501 }
 502
 503 bool isGroupSegment(const GlobalValue *GV) {
 504   return GV->getType()->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
 505 }
 506
 507 bool isGlobalSegment(const GlobalValue *GV) {
 508   return GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
 509 }
 510
 511 bool isReadOnlySegment(const GlobalValue *GV) {
 512   return GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
 513          GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
 514 }
 515
 516 bool shouldEmitConstantsToTextSection(const Triple &TT) {
 517   return TT.getOS() == Triple::AMDPAL;
 518 }
 519
 520 int getIntegerAttribute(const Function &F, StringRef Name, int Default) {
 521   Attribute A = F.getFnAttribute(Name);
 522   int Result = Default;
 523
 524   if (A.isStringAttribute()) {
 525     StringRef Str = A.getValueAsString();
 526     if (Str.getAsInteger(0, Result)) {
 527       LLVMContext &Ctx = F.getContext();
 528       Ctx.emitError("can't parse integer attribute " + Name);
 529     }
 530   }
 531
 532   return Result;
 533 }
 534
 535 std::pair<int, int> getIntegerPairAttribute(const Function &F,
 536                                             StringRef Name,
 537                                             std::pair<int, int> Default,
 538                                             bool OnlyFirstRequired) {
 539   Attribute A = F.getFnAttribute(Name);
 540   if (!A.isStringAttribute())
 541     return Default;
 542
 543   LLVMContext &Ctx = F.getContext();
 544   std::pair<int, int> Ints = Default;
 545   std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
 546   if (Strs.first.trim().getAsInteger(0, Ints.first)) {
 547     Ctx.emitError("can't parse first integer attribute " + Name);
 548     return Default;
 549   }
 550   if (Strs.second.trim().getAsInteger(0, Ints.second)) {
 551     if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
 552       Ctx.emitError("can't parse second integer attribute " + Name);
 553       return Default;
 554     }
 555   }
 556
 557   return Ints;
 558 }
 559
 560 unsigned getVmcntBitMask(const IsaVersion &Version) {
 561   unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1;
 562   if (Version.Major < 9)
 563     return VmcntLo;
 564
 565   unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo();
 566   return VmcntLo | VmcntHi;
 567 }
 568
 569 unsigned getExpcntBitMask(const IsaVersion &Version) {
 570   return (1 << getExpcntBitWidth()) - 1;
 571 }
 572
 573 unsigned getLgkmcntBitMask(const IsaVersion &Version) {
 574   return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
 575 }
 576
 577 unsigned getWaitcntBitMask(const IsaVersion &Version) {
 578   unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo());
 579   unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth());
 580   unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(),
 581                                 getLgkmcntBitWidth(Version.Major));
 582   unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt;
 583   if (Version.Major < 9)
 584     return Waitcnt;
 585
 586   unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi());
 587   return Waitcnt | VmcntHi;
 588 }
 589
 590 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
 591   unsigned VmcntLo =
 592       unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
 593   if (Version.Major < 9)
 594     return VmcntLo;
 595
 596   unsigned VmcntHi =
 597       unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
 598   VmcntHi <<= getVmcntBitWidthLo();
 599   return VmcntLo | VmcntHi;
 600 }
 601
 602 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
 603   return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
 604 }
 605
 606 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
 607   return unpackBits(Waitcnt, getLgkmcntBitShift(),
 608                     getLgkmcntBitWidth(Version.Major));
 609 }
 610
 611 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt,
 612                    unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) {
 613   Vmcnt = decodeVmcnt(Version, Waitcnt);
 614   Expcnt = decodeExpcnt(Version, Waitcnt);
 615   Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
 616 }
 617
 618 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) {
 619   Waitcnt Decoded;
 620   Decoded.VmCnt = decodeVmcnt(Version, Encoded);
 621   Decoded.ExpCnt = decodeExpcnt(Version, Encoded);
 622   Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded);
 623   return Decoded;
 624 }
 625
 626 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
 627                      unsigned Vmcnt) {
 628   Waitcnt =
 629       packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo());
 630   if (Version.Major < 9)
 631     return Waitcnt;
 632
 633   Vmcnt >>= getVmcntBitWidthLo();
 634   return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi());
 635 }
 636
 637 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
 638                       unsigned Expcnt) {
 639   return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth());
 640 }
 641
 642 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
 643                        unsigned Lgkmcnt) {
 644   return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(),
 645                                     getLgkmcntBitWidth(Version.Major));
 646 }
 647
 648 unsigned encodeWaitcnt(const IsaVersion &Version,
 649                        unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) {
 650   unsigned Waitcnt = getWaitcntBitMask(Version);
 651   Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
 652   Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
 653   Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
 654   return Waitcnt;
 655 }
 656
 657 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) {
 658   return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt);
 659 }
 660
 661 //===----------------------------------------------------------------------===//
 662 // hwreg
 663 //===----------------------------------------------------------------------===//
 664
 665 namespace Hwreg {
 666
 667 int64_t getHwregId(const StringRef Name) {
 668   for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) {
 669     if (IdSymbolic[Id] && Name == IdSymbolic[Id])
 670       return Id;
 671   }
 672   return ID_UNKNOWN_;
 673 }
 674
 675 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) {
 676   if (isSI(STI) || isCI(STI) || isVI(STI))
 677     return ID_SYMBOLIC_FIRST_GFX9_;
 678   else if (isGFX9(STI))
 679     return ID_SYMBOLIC_FIRST_GFX10_;
 680   else
 681     return ID_SYMBOLIC_LAST_;
 682 }
 683
 684 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) {
 685   return ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) &&
 686          IdSymbolic[Id];
 687 }
 688
 689 bool isValidHwreg(int64_t Id) {
 690   return 0 <= Id && isUInt<ID_WIDTH_>(Id);
 691 }
 692
 693 bool isValidHwregOffset(int64_t Offset) {
 694   return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset);
 695 }
 696
 697 bool isValidHwregWidth(int64_t Width) {
 698   return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1);
 699 }
 700
 701 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) {
 702   return (Id << ID_SHIFT_) |
 703          (Offset << OFFSET_SHIFT_) |
 704          ((Width - 1) << WIDTH_M1_SHIFT_);
 705 }
 706
 707 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) {
 708   return isValidHwreg(Id, STI) ? IdSymbolic[Id] : "";
 709 }
 710
 711 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) {
 712   Id = (Val & ID_MASK_) >> ID_SHIFT_;
 713   Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_;
 714   Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1;
 715 }
 716
 717 } // namespace Hwreg
 718
 719 //===----------------------------------------------------------------------===//
 720 // SendMsg
 721 //===----------------------------------------------------------------------===//
 722
 723 namespace SendMsg {
 724
 725 int64_t getMsgId(const StringRef Name) {
 726   for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
 727     if (IdSymbolic[i] && Name == IdSymbolic[i])
 728       return i;
 729   }
 730   return ID_UNKNOWN_;
 731 }
 732
 733 static bool isValidMsgId(int64_t MsgId) {
 734   return (ID_GAPS_FIRST_ <= MsgId && MsgId < ID_GAPS_LAST_) && IdSymbolic[MsgId];
 735 }
 736
 737 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) {
 738   if (Strict) {
 739     if (MsgId == ID_GS_ALLOC_REQ || MsgId == ID_GET_DOORBELL)
 740       return isGFX9(STI) || isGFX10(STI);
 741     else
 742       return isValidMsgId(MsgId);
 743   } else {
 744     return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId);
 745   }
 746 }
 747
 748 StringRef getMsgName(int64_t MsgId) {
 749   return isValidMsgId(MsgId)? IdSymbolic[MsgId] : "";
 750 }
 751
 752 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) {
 753   const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
 754   const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
 755   const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
 756   for (int i = F; i < L; ++i) {
 757     if (Name == S[i]) {
 758       return i;
 759     }
 760   }
 761   return OP_UNKNOWN_;
 762 }
 763
 764 bool isValidMsgOp(int64_t MsgId, int64_t OpId, bool Strict) {
 765
 766   if (!Strict)
 767     return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
 768
 769   switch(MsgId)
 770   {
 771   case ID_GS:
 772     return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP;
 773   case ID_GS_DONE:
 774     return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_;
 775   case ID_SYSMSG:
 776     return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_;
 777   default:
 778     return OpId == OP_NONE_;
 779   }
 780 }
 781
 782 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) {
 783   assert(msgRequiresOp(MsgId));
 784   return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId];
 785 }
 786
 787 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, bool Strict) {
 788
 789   if (!Strict)
 790     return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);
 791
 792   switch(MsgId)
 793   {
 794   case ID_GS:
 795     return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
 796   case ID_GS_DONE:
 797     return (OpId == OP_GS_NOP)?
 798            (StreamId == STREAM_ID_NONE_) :
 799            (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
 800   default:
 801     return StreamId == STREAM_ID_NONE_;
 802   }
 803 }
 804
 805 bool msgRequiresOp(int64_t MsgId) {
 806   return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG;
 807 }
 808
 809 bool msgSupportsStream(int64_t MsgId, int64_t OpId) {
 810   return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP;
 811 }
 812
 813 void decodeMsg(unsigned Val,
 814                uint16_t &MsgId,
 815                uint16_t &OpId,
 816                uint16_t &StreamId) {
 817   MsgId = Val & ID_MASK_;
 818   OpId = (Val & OP_MASK_) >> OP_SHIFT_;
 819   StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
 820 }
 821
 822 uint64_t encodeMsg(uint64_t MsgId,
 823                    uint64_t OpId,
 824                    uint64_t StreamId) {
 825   return (MsgId << ID_SHIFT_) |
 826          (OpId << OP_SHIFT_) |
 827          (StreamId << STREAM_ID_SHIFT_);
 828 }
 829
 830 } // namespace SendMsg
 831
 832 //===----------------------------------------------------------------------===//
 833 //
 834 //===----------------------------------------------------------------------===//
 835
 836 unsigned getInitialPSInputAddr(const Function &F) {
 837   return getIntegerAttribute(F, "InitialPSInputAddr", 0);
 838 }
 839
 840 bool isShader(CallingConv::ID cc) {
 841   switch(cc) {
 842     case CallingConv::AMDGPU_VS:
 843     case CallingConv::AMDGPU_LS:
 844     case CallingConv::AMDGPU_HS:
 845     case CallingConv::AMDGPU_ES:
 846     case CallingConv::AMDGPU_GS:
 847     case CallingConv::AMDGPU_PS:
 848     case CallingConv::AMDGPU_CS:
 849       return true;
 850     default:
 851       return false;
 852   }
 853 }
 854
 855 bool isCompute(CallingConv::ID cc) {
 856   return !isShader(cc) || cc == CallingConv::AMDGPU_CS;
 857 }
 858
 859 bool isEntryFunctionCC(CallingConv::ID CC) {
 860   switch (CC) {
 861   case CallingConv::AMDGPU_KERNEL:
 862   case CallingConv::SPIR_KERNEL:
 863   case CallingConv::AMDGPU_VS:
 864   case CallingConv::AMDGPU_GS:
 865   case CallingConv::AMDGPU_PS:
 866   case CallingConv::AMDGPU_CS:
 867   case CallingConv::AMDGPU_ES:
 868   case CallingConv::AMDGPU_HS:
 869   case CallingConv::AMDGPU_LS:
 870     return true;
 871   default:
 872     return false;
 873   }
 874 }
 875
 876 bool hasXNACK(const MCSubtargetInfo &STI) {
 877   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
 878 }
 879
 880 bool hasSRAMECC(const MCSubtargetInfo &STI) {
 881   return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC];
 882 }
 883
 884 bool hasMIMG_R128(const MCSubtargetInfo &STI) {
 885   return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128];
 886 }
 887
 888 bool hasPackedD16(const MCSubtargetInfo &STI) {
 889   return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem];
 890 }
 891
 892 bool isSI(const MCSubtargetInfo &STI) {
 893   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
 894 }
 895
 896 bool isCI(const MCSubtargetInfo &STI) {
 897   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
 898 }
 899
 900 bool isVI(const MCSubtargetInfo &STI) {
 901   return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands];
 902 }
 903
 904 bool isGFX9(const MCSubtargetInfo &STI) {
 905   return STI.getFeatureBits()[AMDGPU::FeatureGFX9];
 906 }
 907
 908 bool isGFX10(const MCSubtargetInfo &STI) {
 909   return STI.getFeatureBits()[AMDGPU::FeatureGFX10];
 910 }
 911
 912 bool isGCN3Encoding(const MCSubtargetInfo &STI) {
 913   return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding];
 914 }
 915
 916 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) {
 917   const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID);
 918   const unsigned FirstSubReg = TRI->getSubReg(Reg, 1);
 919   return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
 920     Reg == AMDGPU::SCC;
 921 }
 922
 923 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) {
 924   for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) {
 925     if (*R == Reg1) return true;
 926   }
 927   return false;
 928 }
 929
 930 #define MAP_REG2REG \
 931   using namespace AMDGPU; \
 932   switch(Reg) { \
 933   default: return Reg; \
 934   CASE_CI_VI(FLAT_SCR) \
 935   CASE_CI_VI(FLAT_SCR_LO) \
 936   CASE_CI_VI(FLAT_SCR_HI) \
 937   CASE_VI_GFX9_GFX10(TTMP0) \
 938   CASE_VI_GFX9_GFX10(TTMP1) \
 939   CASE_VI_GFX9_GFX10(TTMP2) \
 940   CASE_VI_GFX9_GFX10(TTMP3) \
 941   CASE_VI_GFX9_GFX10(TTMP4) \
 942   CASE_VI_GFX9_GFX10(TTMP5) \
 943   CASE_VI_GFX9_GFX10(TTMP6) \
 944   CASE_VI_GFX9_GFX10(TTMP7) \
 945   CASE_VI_GFX9_GFX10(TTMP8) \
 946   CASE_VI_GFX9_GFX10(TTMP9) \
 947   CASE_VI_GFX9_GFX10(TTMP10) \
 948   CASE_VI_GFX9_GFX10(TTMP11) \
 949   CASE_VI_GFX9_GFX10(TTMP12) \
 950   CASE_VI_GFX9_GFX10(TTMP13) \
 951   CASE_VI_GFX9_GFX10(TTMP14) \
 952   CASE_VI_GFX9_GFX10(TTMP15) \
 953   CASE_VI_GFX9_GFX10(TTMP0_TTMP1) \
 954   CASE_VI_GFX9_GFX10(TTMP2_TTMP3) \
 955   CASE_VI_GFX9_GFX10(TTMP4_TTMP5) \
 956   CASE_VI_GFX9_GFX10(TTMP6_TTMP7) \
 957   CASE_VI_GFX9_GFX10(TTMP8_TTMP9) \
 958   CASE_VI_GFX9_GFX10(TTMP10_TTMP11) \
 959   CASE_VI_GFX9_GFX10(TTMP12_TTMP13) \
 960   CASE_VI_GFX9_GFX10(TTMP14_TTMP15) \
 961   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3) \
 962   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7) \
 963   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11) \
 964   CASE_VI_GFX9_GFX10(TTMP12_TTMP13_TTMP14_TTMP15) \
 965   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
 966   CASE_VI_GFX9_GFX10(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
 967   CASE_VI_GFX9_GFX10(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
 968   CASE_VI_GFX9_GFX10(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
 969   }
 970
 971 #define CASE_CI_VI(node) \
 972   assert(!isSI(STI)); \
 973   case node: return isCI(STI) ? node##_ci : node##_vi;
 974
 975 #define CASE_VI_GFX9_GFX10(node) \
 976   case node: return (isGFX9(STI) || isGFX10(STI)) ? node##_gfx9_gfx10 : node##_vi;
 977
 978 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) {
 979   if (STI.getTargetTriple().getArch() == Triple::r600)
 980     return Reg;
 981   MAP_REG2REG
 982 }
 983
 984 #undef CASE_CI_VI
 985 #undef CASE_VI_GFX9_GFX10
 986
 987 #define CASE_CI_VI(node)   case node##_ci: case node##_vi:   return node;
 988 #define CASE_VI_GFX9_GFX10(node) case node##_vi: case node##_gfx9_gfx10: return node;
 989
 990 unsigned mc2PseudoReg(unsigned Reg) {
 991   MAP_REG2REG
 992 }
 993
 994 #undef CASE_CI_VI
 995 #undef CASE_VI_GFX9_GFX10
 996 #undef MAP_REG2REG
 997
 998 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
 999   assert(OpNo < Desc.NumOperands);
1000   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1001   return OpType >= AMDGPU::OPERAND_SRC_FIRST &&
1002          OpType <= AMDGPU::OPERAND_SRC_LAST;
1003 }
1004
1005 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1006   assert(OpNo < Desc.NumOperands);
1007   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1008   switch (OpType) {
1009   case AMDGPU::OPERAND_REG_IMM_FP32:
1010   case AMDGPU::OPERAND_REG_IMM_FP64:
1011   case AMDGPU::OPERAND_REG_IMM_FP16:
1012   case AMDGPU::OPERAND_REG_IMM_V2FP16:
1013   case AMDGPU::OPERAND_REG_IMM_V2INT16:
1014   case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1015   case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1016   case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1017   case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1018   case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1019   case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1020   case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1021   case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1022   case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1023     return true;
1024   default:
1025     return false;
1026   }
1027 }
1028
1029 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1030   assert(OpNo < Desc.NumOperands);
1031   unsigned OpType = Desc.OpInfo[OpNo].OperandType;
1032   return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
1033          OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST;
1034 }
1035
1036 // Avoid using MCRegisterClass::getSize, since that function will go away
1037 // (move from MC* level to Target* level). Return size in bits.
1038 unsigned getRegBitWidth(unsigned RCID) {
1039   switch (RCID) {
1040   case AMDGPU::SGPR_32RegClassID:
1041   case AMDGPU::VGPR_32RegClassID:
1042   case AMDGPU::VRegOrLds_32RegClassID:
1043   case AMDGPU::AGPR_32RegClassID:
1044   case AMDGPU::VS_32RegClassID:
1045   case AMDGPU::AV_32RegClassID:
1046   case AMDGPU::SReg_32RegClassID:
1047   case AMDGPU::SReg_32_XM0RegClassID:
1048   case AMDGPU::SRegOrLds_32RegClassID:
1049     return 32;
1050   case AMDGPU::SGPR_64RegClassID:
1051   case AMDGPU::VS_64RegClassID:
1052   case AMDGPU::AV_64RegClassID:
1053   case AMDGPU::SReg_64RegClassID:
1054   case AMDGPU::VReg_64RegClassID:
1055   case AMDGPU::AReg_64RegClassID:
1056   case AMDGPU::SReg_64_XEXECRegClassID:
1057     return 64;
1058   case AMDGPU::SGPR_96RegClassID:
1059   case AMDGPU::SReg_96RegClassID:
1060   case AMDGPU::VReg_96RegClassID:
1061     return 96;
1062   case AMDGPU::SGPR_128RegClassID:
1063   case AMDGPU::SReg_128RegClassID:
1064   case AMDGPU::VReg_128RegClassID:
1065   case AMDGPU::AReg_128RegClassID:
1066     return 128;
1067   case AMDGPU::SGPR_160RegClassID:
1068   case AMDGPU::SReg_160RegClassID:
1069   case AMDGPU::VReg_160RegClassID:
1070     return 160;
1071   case AMDGPU::SReg_256RegClassID:
1072   case AMDGPU::VReg_256RegClassID:
1073     return 256;
1074   case AMDGPU::SReg_512RegClassID:
1075   case AMDGPU::VReg_512RegClassID:
1076   case AMDGPU::AReg_512RegClassID:
1077     return 512;
1078   case AMDGPU::SReg_1024RegClassID:
1079   case AMDGPU::VReg_1024RegClassID:
1080   case AMDGPU::AReg_1024RegClassID:
1081     return 1024;
1082   default:
1083     llvm_unreachable("Unexpected register class");
1084   }
1085 }
1086
1087 unsigned getRegBitWidth(const MCRegisterClass &RC) {
1088   return getRegBitWidth(RC.getID());
1089 }
1090
1091 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc,
1092                            unsigned OpNo) {
1093   assert(OpNo < Desc.NumOperands);
1094   unsigned RCID = Desc.OpInfo[OpNo].RegClass;
1095   return getRegBitWidth(MRI->getRegClass(RCID)) / 8;
1096 }
1097
1098 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
1099   if (Literal >= -16 && Literal <= 64)
1100     return true;
1101
1102   uint64_t Val = static_cast<uint64_t>(Literal);
1103   return (Val == DoubleToBits(0.0)) ||
1104          (Val == DoubleToBits(1.0)) ||
1105          (Val == DoubleToBits(-1.0)) ||
1106          (Val == DoubleToBits(0.5)) ||
1107          (Val == DoubleToBits(-0.5)) ||
1108          (Val == DoubleToBits(2.0)) ||
1109          (Val == DoubleToBits(-2.0)) ||
1110          (Val == DoubleToBits(4.0)) ||
1111          (Val == DoubleToBits(-4.0)) ||
1112          (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
1113 }
1114
1115 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
1116   if (Literal >= -16 && Literal <= 64)
1117     return true;
1118
1119   // The actual type of the operand does not seem to matter as long
1120   // as the bits match one of the inline immediate values.  For example:
1121   //
1122   // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
1123   // so it is a legal inline immediate.
1124   //
1125   // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
1126   // floating-point, so it is a legal inline immediate.
1127
1128   uint32_t Val = static_cast<uint32_t>(Literal);
1129   return (Val == FloatToBits(0.0f)) ||
1130          (Val == FloatToBits(1.0f)) ||
1131          (Val == FloatToBits(-1.0f)) ||
1132          (Val == FloatToBits(0.5f)) ||
1133          (Val == FloatToBits(-0.5f)) ||
1134          (Val == FloatToBits(2.0f)) ||
1135          (Val == FloatToBits(-2.0f)) ||
1136          (Val == FloatToBits(4.0f)) ||
1137          (Val == FloatToBits(-4.0f)) ||
1138          (Val == 0x3e22f983 && HasInv2Pi);
1139 }
1140
1141 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) {
1142   if (!HasInv2Pi)
1143     return false;
1144
1145   if (Literal >= -16 && Literal <= 64)
1146     return true;
1147
1148   uint16_t Val = static_cast<uint16_t>(Literal);
1149   return Val == 0x3C00 || // 1.0
1150          Val == 0xBC00 || // -1.0
1151          Val == 0x3800 || // 0.5
1152          Val == 0xB800 || // -0.5
1153          Val == 0x4000 || // 2.0
1154          Val == 0xC000 || // -2.0
1155          Val == 0x4400 || // 4.0
1156          Val == 0xC400 || // -4.0
1157          Val == 0x3118;   // 1/2pi
1158 }
1159
1160 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) {
1161   assert(HasInv2Pi);
1162
1163   if (isInt<16>(Literal) || isUInt<16>(Literal)) {
1164     int16_t Trunc = static_cast<int16_t>(Literal);
1165     return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi);
1166   }
1167   if (!(Literal & 0xffff))
1168     return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi);
1169
1170   int16_t Lo16 = static_cast<int16_t>(Literal);
1171   int16_t Hi16 = static_cast<int16_t>(Literal >> 16);
1172   return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi);
1173 }
1174
1175 bool isArgPassedInSGPR(const Argument *A) {
1176   const Function *F = A->getParent();
1177
1178   // Arguments to compute shaders are never a source of divergence.
1179   CallingConv::ID CC = F->getCallingConv();
1180   switch (CC) {
1181   case CallingConv::AMDGPU_KERNEL:
1182   case CallingConv::SPIR_KERNEL:
1183     return true;
1184   case CallingConv::AMDGPU_VS:
1185   case CallingConv::AMDGPU_LS:
1186   case CallingConv::AMDGPU_HS:
1187   case CallingConv::AMDGPU_ES:
1188   case CallingConv::AMDGPU_GS:
1189   case CallingConv::AMDGPU_PS:
1190   case CallingConv::AMDGPU_CS:
1191     // For non-compute shaders, SGPR inputs are marked with either inreg or byval.
1192     // Everything else is in VGPRs.
1193     return F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::InReg) ||
1194            F->getAttributes().hasParamAttribute(A->getArgNo(), Attribute::ByVal);
1195   default:
1196     // TODO: Should calls support inreg for SGPR inputs?
1197     return false;
1198   }
1199 }
1200
1201 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
1202   return isGCN3Encoding(ST) || isGFX10(ST);
1203 }
1204
1205 int64_t getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
1206   if (hasSMEMByteOffset(ST))
1207     return ByteOffset;
1208   return ByteOffset >> 2;
1209 }
1210
1211 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset) {
1212   int64_t EncodedOffset = getSMRDEncodedOffset(ST, ByteOffset);
1213   return (hasSMEMByteOffset(ST)) ?
1214     isUInt<20>(EncodedOffset) : isUInt<8>(EncodedOffset);
1215 }
1216
1217 // Given Imm, split it into the values to put into the SOffset and ImmOffset
1218 // fields in an MUBUF instruction. Return false if it is not possible (due to a
1219 // hardware bug needing a workaround).
1220 //
1221 // The required alignment ensures that individual address components remain
1222 // aligned if they are aligned to begin with. It also ensures that additional
1223 // offsets within the given alignment can be added to the resulting ImmOffset.
1224 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1225                       const GCNSubtarget *Subtarget, uint32_t Align) {
1226   const uint32_t MaxImm = alignDown(4095, Align);
1227   uint32_t Overflow = 0;
1228
1229   if (Imm > MaxImm) {
1230     if (Imm <= MaxImm + 64) {
1231       // Use an SOffset inline constant for 4..64
1232       Overflow = Imm - MaxImm;
1233       Imm = MaxImm;
1234     } else {
1235       // Try to keep the same value in SOffset for adjacent loads, so that
1236       // the corresponding register contents can be re-used.
1237       //
1238       // Load values with all low-bits (except for alignment bits) set into
1239       // SOffset, so that a larger range of values can be covered using
1240       // s_movk_i32.
1241       //
1242       // Atomic operations fail to work correctly when individual address
1243       // components are unaligned, even if their sum is aligned.
1244       uint32_t High = (Imm + Align) & ~4095;
1245       uint32_t Low = (Imm + Align) & 4095;
1246       Imm = Low;
1247       Overflow = High - Align;
1248     }
1249   }
1250
1251   // There is a hardware bug in SI and CI which prevents address clamping in
1252   // MUBUF instructions from working correctly with SOffsets. The immediate
1253   // offset is unaffected.
1254   if (Overflow > 0 &&
1255       Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
1256     return false;
1257
1258   ImmOffset = Imm;
1259   SOffset = Overflow;
1260   return true;
1261 }
1262
1263 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
1264   *this = getDefaultForCallingConv(F.getCallingConv());
1265
1266   StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
1267   if (!IEEEAttr.empty())
1268     IEEE = IEEEAttr == "true";
1269
1270   StringRef DX10ClampAttr
1271     = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
1272   if (!DX10ClampAttr.empty())
1273     DX10Clamp = DX10ClampAttr == "true";
1274 }
1275
1276 namespace {
1277
1278 struct SourceOfDivergence {
1279   unsigned Intr;
1280 };
1281 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
1282
1283 #define GET_SourcesOfDivergence_IMPL
1284 #include "AMDGPUGenSearchableTables.inc"
1285
1286 } // end anonymous namespace
1287
1288 bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
1289   return lookupSourceOfDivergence(IntrID);
1290 }
1291
1292 } // namespace AMDGPU
1293 } // namespace llvm