llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h

   1 //===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //==-----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// Defines an instruction selector for the AMDGPU target.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
  15 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
  16
  17 #include "GCNSubtarget.h"
  18 #include "SIMachineFunctionInfo.h"
  19 #include "SIModeRegisterDefaults.h"
  20 #include "llvm/CodeGen/SelectionDAGISel.h"
  21 #include "llvm/Target/TargetMachine.h"
  22
  23 using namespace llvm;
  24
  25 namespace {
  26
  27 static inline bool getConstantValue(SDValue N, uint32_t &Out) {
  28   // This is only used for packed vectors, where using 0 for undef should
  29   // always be good.
  30   if (N.isUndef()) {
  31     Out = 0;
  32     return true;
  33   }
  34
  35   if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
  36     Out = C->getAPIntValue().getSExtValue();
  37     return true;
  38   }
  39
  40   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
  41     Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
  42     return true;
  43   }
  44
  45   return false;
  46 }
  47
  48 // TODO: Handle undef as zero
  49 static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
  50   assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
  51   uint32_t LHSVal, RHSVal;
  52   if (getConstantValue(N->getOperand(0), LHSVal) &&
  53       getConstantValue(N->getOperand(1), RHSVal)) {
  54     SDLoc SL(N);
  55     uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
  56     return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
  57                               DAG.getTargetConstant(K, SL, MVT::i32));
  58   }
  59
  60   return nullptr;
  61 }
  62
  63 } // namespace
  64
  65 /// AMDGPU specific code to select AMDGPU machine instructions for
  66 /// SelectionDAG operations.
  67 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
  68   // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
  69   // make the right decision when generating code for different targets.
  70   const GCNSubtarget *Subtarget;
  71
  72   // Default FP mode for the current function.
  73   SIModeRegisterDefaults Mode;
  74
  75   bool EnableLateStructurizeCFG;
  76
  77   // Instructions that will be lowered with a final instruction that zeros the
  78   // high result bits.
  79   bool fp16SrcZerosHighBits(unsigned Opc) const;
  80
  81 public:
  82   AMDGPUDAGToDAGISel() = delete;
  83
  84   explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);
  85
  86   bool runOnMachineFunction(MachineFunction &MF) override;
  87   bool matchLoadD16FromBuildVector(SDNode *N) const;
  88   void PreprocessISelDAG() override;
  89   void Select(SDNode *N) override;
  90   void PostprocessISelDAG() override;
  91
  92 protected:
  93   void SelectBuildVector(SDNode *N, unsigned RegClassID);
  94
  95 private:
  96   std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
  97
  98   bool isInlineImmediate(const SDNode *N) const;
  99
 100   bool isInlineImmediate(const APInt &Imm) const {
 101     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
 102   }
 103
 104   bool isInlineImmediate(const APFloat &Imm) const {
 105     return Subtarget->getInstrInfo()->isInlineConstant(Imm);
 106   }
 107
 108   bool isVGPRImm(const SDNode *N) const;
 109   bool isUniformLoad(const SDNode *N) const;
 110   bool isUniformBr(const SDNode *N) const;
 111
 112   // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
 113   // `ShAmtBits` bits is unneeded.
 114   bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
 115
 116   bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
 117                                   SDValue &RHS) const;
 118
 119   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
 120
 121   SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
 122   SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
 123   SDNode *glueCopyToM0LDSInit(SDNode *N) const;
 124
 125   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
 126   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
 127   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
 128   bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
 129   bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
 130                         unsigned Size) const;
 131
 132   bool isFlatScratchBaseLegal(SDValue Addr) const;
 133   bool isFlatScratchBaseLegalSV(SDValue Addr) const;
 134   bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
 135   bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,
 136                                    bool IsBuffer, int64_t ImmOffset = 0) const;
 137
 138   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
 139   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
 140                                  SDValue &Offset1) const;
 141   bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
 142                                   SDValue &Offset1) const;
 143   bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
 144                           SDValue &Offset1, unsigned Size) const;
 145   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
 146                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
 147                    SDValue &Idxen, SDValue &Addr64) const;
 148   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
 149                          SDValue &SOffset, SDValue &Offset) const;
 150   bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
 151                                SDValue &VAddr, SDValue &SOffset,
 152                                SDValue &ImmOffset) const;
 153   bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
 154                                 SDValue &Soffset, SDValue &Offset) const;
 155
 156   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
 157                          SDValue &Offset) const;
 158   bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
 159
 160   bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
 161                             SDValue &Offset, uint64_t FlatVariant) const;
 162   bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
 163                         SDValue &Offset) const;
 164   bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
 165                           SDValue &Offset) const;
 166   bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
 167                            SDValue &Offset) const;
 168   bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
 169                          SDValue &VOffset, SDValue &Offset) const;
 170   bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
 171                           SDValue &Offset) const;
 172   bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
 173                                      uint64_t ImmOffset) const;
 174   bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
 175                            SDValue &SAddr, SDValue &Offset) const;
 176
 177   bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue *SOffset,
 178                         SDValue *Offset, bool Imm32Only = false,
 179                         bool IsBuffer = false, bool HasSOffset = false,
 180                         int64_t ImmOffset = 0) const;
 181   SDValue Expand32BitAddress(SDValue Addr) const;
 182   bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset,
 183                             SDValue *Offset, bool Imm32Only = false,
 184                             bool IsBuffer = false, bool HasSOffset = false,
 185                             int64_t ImmOffset = 0) const;
 186   bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset,
 187                   SDValue *Offset, bool Imm32Only = false) const;
 188   bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
 189   bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
 190   bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const;
 191   bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset,
 192                          SDValue &Offset) const;
 193   bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
 194   bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
 195   bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
 196                                SDValue &Offset) const;
 197   bool SelectSMRDPrefetchImm(SDValue Addr, SDValue &SBase,
 198                              SDValue &Offset) const;
 199   bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
 200
 201   bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
 202                           bool IsCanonicalizing = true,
 203                           bool AllowAbs = true) const;
 204   bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 205   bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
 206                                        SDValue &SrcMods) const;
 207   bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 208   bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
 209   bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
 210                        SDValue &Clamp, SDValue &Omod) const;
 211   bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
 212                         SDValue &Clamp, SDValue &Omod) const;
 213   bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
 214                          SDValue &Clamp, SDValue &Omod) const;
 215
 216   bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
 217                              bool OpSel) const;
 218   bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 219   bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 220
 221   bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
 222                        SDValue &Omod) const;
 223
 224   bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
 225                        bool IsDOT = false) const;
 226   bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 227
 228   bool SelectVOP3PModsNeg(SDValue In, SDValue &Src) const;
 229   bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
 230
 231   bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
 232                                SDValue &SrcMods) const;
 233   bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 234   bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
 235                                SDValue &SrcMods) const;
 236   bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
 237
 238   bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
 239   bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
 240
 241   bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 242
 243   bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 244   bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
 245                                  unsigned &Mods) const;
 246   bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
 247                                 SDValue &SrcMods) const;
 248   bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
 249
 250   SDValue getHi16Elt(SDValue In) const;
 251
 252   SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
 253
 254   void SelectADD_SUB_I64(SDNode *N);
 255   void SelectAddcSubb(SDNode *N);
 256   void SelectUADDO_USUBO(SDNode *N);
 257   void SelectDIV_SCALE(SDNode *N);
 258   void SelectMAD_64_32(SDNode *N);
 259   void SelectMUL_LOHI(SDNode *N);
 260   void SelectFMA_W_CHAIN(SDNode *N);
 261   void SelectFMUL_W_CHAIN(SDNode *N);
 262   SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
 263                    uint32_t Width);
 264   void SelectS_BFEFromShifts(SDNode *N);
 265   void SelectS_BFE(SDNode *N);
 266   bool isCBranchSCC(const SDNode *N) const;
 267   void SelectBRCOND(SDNode *N);
 268   void SelectFMAD_FMA(SDNode *N);
 269   void SelectFP_EXTEND(SDNode *N);
 270   void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
 271   void SelectDSBvhStackIntrinsic(SDNode *N);
 272   void SelectDS_GWS(SDNode *N, unsigned IntrID);
 273   void SelectInterpP1F16(SDNode *N);
 274   void SelectINTRINSIC_W_CHAIN(SDNode *N);
 275   void SelectINTRINSIC_WO_CHAIN(SDNode *N);
 276   void SelectINTRINSIC_VOID(SDNode *N);
 277   void SelectWAVE_ADDRESS(SDNode *N);
 278   void SelectSTACKRESTORE(SDNode *N);
 279
 280 protected:
 281   // Include the pieces autogenerated from the target description.
 282 #include "AMDGPUGenDAGISel.inc"
 283 };
 284
 285 class AMDGPUISelDAGToDAGPass : public SelectionDAGISelPass {
 286 public:
 287   AMDGPUISelDAGToDAGPass(TargetMachine &TM);
 288
 289   PreservedAnalyses run(MachineFunction &MF,
 290                         MachineFunctionAnalysisManager &MFAM);
 291 };
 292
 293 class AMDGPUDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
 294 public:
 295   static char ID;
 296
 297   AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel);
 298
 299   bool runOnMachineFunction(MachineFunction &MF) override;
 300   void getAnalysisUsage(AnalysisUsage &AU) const override;
 301   StringRef getPassName() const override;
 302 };
 303
 304 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H