lib/Target/AMDGPU/SIISelLowering.h

   1 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// SI DAG Lowering interface definition
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
  15 #define LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
  16
  17 #include "AMDGPUISelLowering.h"
  18 #include "AMDGPUArgumentUsageInfo.h"
  19 #include "SIInstrInfo.h"
  20
  21 namespace llvm {
  22
  23 class SITargetLowering final : public AMDGPUTargetLowering {
  24 private:
  25   const GCNSubtarget *Subtarget;
  26
  27 public:
  28   MVT getRegisterTypeForCallingConv(LLVMContext &Context,
  29                                     CallingConv::ID CC,
  30                                     EVT VT) const override;
  31   unsigned getNumRegistersForCallingConv(LLVMContext &Context,
  32                                          CallingConv::ID CC,
  33                                          EVT VT) const override;
  34
  35   unsigned getVectorTypeBreakdownForCallingConv(
  36     LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
  37     unsigned &NumIntermediates, MVT &RegisterVT) const override;
  38
  39 private:
  40   SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL,
  41                                    SDValue Chain, uint64_t Offset) const;
  42   SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const;
  43   SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
  44                                    const SDLoc &SL, SDValue Chain,
  45                                    uint64_t Offset, unsigned Align, bool Signed,
  46                                    const ISD::InputArg *Arg = nullptr) const;
  47
  48   SDValue lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
  49                               const SDLoc &SL, SDValue Chain,
  50                               const ISD::InputArg &Arg) const;
  51   SDValue getPreloadedValue(SelectionDAG &DAG,
  52                             const SIMachineFunctionInfo &MFI,
  53                             EVT VT,
  54                             AMDGPUFunctionArgInfo::PreloadedValue) const;
  55
  56   SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op,
  57                              SelectionDAG &DAG) const override;
  58   SDValue lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
  59                                  MVT VT, unsigned Offset) const;
  60   SDValue lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr,
  61                      SelectionDAG &DAG) const;
  62   SDValue lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset,
  63                        SDValue GLC, SDValue DLC, SelectionDAG &DAG) const;
  64
  65   SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
  66   SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
  67   SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
  68
  69   // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
  70   // (the offset that is included in bounds checking and swizzling, to be split
  71   // between the instruction's voffset and immoffset fields) and soffset (the
  72   // offset that is excluded from bounds checking and swizzling, to go in the
  73   // instruction's soffset field).  This function takes the first kind of
  74   // offset and figures out how to split it between voffset and immoffset.
  75   std::pair<SDValue, SDValue> splitBufferOffsets(SDValue Offset,
  76                                                  SelectionDAG &DAG) const;
  77
  78   SDValue widenLoad(LoadSDNode *Ld, DAGCombinerInfo &DCI) const;
  79   SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
  80   SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
  81   SDValue lowerFastUnsafeFDIV(SDValue Op, SelectionDAG &DAG) const;
  82   SDValue lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const;
  83   SDValue LowerFDIV16(SDValue Op, SelectionDAG &DAG) const;
  84   SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
  85   SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
  86   SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;
  87   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG, bool Signed) const;
  88   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
  89   SDValue LowerTrig(SDValue Op, SelectionDAG &DAG) const;
  90   SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
  91   SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
  92   SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
  93   SDValue adjustLoadValueType(unsigned Opcode, MemSDNode *M,
  94                               SelectionDAG &DAG, ArrayRef<SDValue> Ops,
  95                               bool IsIntrinsic = false) const;
  96
  97   SDValue lowerIntrinsicLoad(MemSDNode *M, bool IsFormat, SelectionDAG &DAG,
  98                              ArrayRef<SDValue> Ops) const;
  99
 100   // Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
 101   // dwordx4 if on SI.
 102   SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
 103                               ArrayRef<SDValue> Ops, EVT MemVT,
 104                               MachineMemOperand *MMO, SelectionDAG &DAG) const;
 105
 106   SDValue handleD16VData(SDValue VData, SelectionDAG &DAG) const;
 107
 108   /// Converts \p Op, which must be of floating point type, to the
 109   /// floating point type \p VT, by either extending or truncating it.
 110   SDValue getFPExtOrFPTrunc(SelectionDAG &DAG,
 111                             SDValue Op,
 112                             const SDLoc &DL,
 113                             EVT VT) const;
 114
 115   SDValue convertArgType(
 116     SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Val,
 117     bool Signed, const ISD::InputArg *Arg = nullptr) const;
 118
 119   /// Custom lowering for ISD::FP_ROUND for MVT::f16.
 120   SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
 121   SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
 122
 123   SDValue getSegmentAperture(unsigned AS, const SDLoc &DL,
 124                              SelectionDAG &DAG) const;
 125
 126   SDValue lowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
 127   SDValue lowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
 128   SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
 129   SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
 130   SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
 131   SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
 132   SDValue lowerTRAP(SDValue Op, SelectionDAG &DAG) const;
 133   SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
 134
 135   SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const;
 136
 137   SDValue performUCharToFloatCombine(SDNode *N,
 138                                      DAGCombinerInfo &DCI) const;
 139   SDValue performSHLPtrCombine(SDNode *N,
 140                                unsigned AS,
 141                                EVT MemVT,
 142                                DAGCombinerInfo &DCI) const;
 143
 144   SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const;
 145
 146   SDValue splitBinaryBitConstantOp(DAGCombinerInfo &DCI, const SDLoc &SL,
 147                                    unsigned Opc, SDValue LHS,
 148                                    const ConstantSDNode *CRHS) const;
 149
 150   SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 151   SDValue performOrCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 152   SDValue performXorCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 153   SDValue performZeroExtendCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 154   SDValue performSignExtendInRegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 155   SDValue performClassCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 156   SDValue getCanonicalConstantFP(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
 157                                  const APFloat &C) const;
 158   SDValue performFCanonicalizeCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 159
 160   SDValue performFPMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
 161                                   SDValue Op0, SDValue Op1) const;
 162   SDValue performIntMed3ImmCombine(SelectionDAG &DAG, const SDLoc &SL,
 163                                    SDValue Op0, SDValue Op1, bool Signed) const;
 164   SDValue performMinMaxCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 165   SDValue performFMed3Combine(SDNode *N, DAGCombinerInfo &DCI) const;
 166   SDValue performCvtPkRTZCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 167   SDValue performExtractVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 168   SDValue performInsertVectorEltCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 169
 170   SDValue reassociateScalarOps(SDNode *N, SelectionDAG &DAG) const;
 171   unsigned getFusedOpcode(const SelectionDAG &DAG,
 172                           const SDNode *N0, const SDNode *N1) const;
 173   SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 174   SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 175   SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 176   SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 177   SDValue performFSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 178   SDValue performFMACombine(SDNode *N, DAGCombinerInfo &DCI) const;
 179   SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 180   SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 181   SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 182   SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
 183
 184   bool isLegalFlatAddressingMode(const AddrMode &AM) const;
 185   bool isLegalMUBUFAddressingMode(const AddrMode &AM) const;
 186
 187   unsigned isCFIntrinsic(const SDNode *Intr) const;
 188
 189 public:
 190   /// \returns True if fixup needs to be emitted for given global value \p GV,
 191   /// false otherwise.
 192   bool shouldEmitFixup(const GlobalValue *GV) const;
 193
 194   /// \returns True if GOT relocation needs to be emitted for given global value
 195   /// \p GV, false otherwise.
 196   bool shouldEmitGOTReloc(const GlobalValue *GV) const;
 197
 198   /// \returns True if PC-relative relocation needs to be emitted for given
 199   /// global value \p GV, false otherwise.
 200   bool shouldEmitPCReloc(const GlobalValue *GV) const;
 201
 202 private:
 203   // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
 204   // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
 205   // pointed to by Offsets.
 206   /// \returns 0 If there is a non-constant offset or if the offset is 0.
 207   /// Otherwise returns the constant offset.
 208   unsigned setBufferOffsets(SDValue CombinedOffset, SelectionDAG &DAG,
 209                            SDValue *Offsets, unsigned Align = 4) const;
 210
 211   // Handle 8 bit and 16 bit buffer loads
 212   SDValue handleByteShortBufferLoads(SelectionDAG &DAG, EVT LoadVT, SDLoc DL,
 213                                      ArrayRef<SDValue> Ops, MemSDNode *M) const;
 214
 215   // Handle 8 bit and 16 bit buffer stores
 216   SDValue handleByteShortBufferStores(SelectionDAG &DAG, EVT VDataType,
 217                                       SDLoc DL, SDValue Ops[],
 218                                       MemSDNode *M) const;
 219
 220 public:
 221   SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI);
 222
 223   const GCNSubtarget *getSubtarget() const;
 224
 225   bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const override;
 226
 227   bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override;
 228
 229   bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
 230                           MachineFunction &MF,
 231                           unsigned IntrinsicID) const override;
 232
 233   bool getAddrModeArguments(IntrinsicInst * /*I*/,
 234                             SmallVectorImpl<Value*> &/*Ops*/,
 235                             Type *&/*AccessTy*/) const override;
 236
 237   bool isLegalGlobalAddressingMode(const AddrMode &AM) const;
 238   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
 239                              unsigned AS,
 240                              Instruction *I = nullptr) const override;
 241
 242   bool canMergeStoresTo(unsigned AS, EVT MemVT,
 243                         const SelectionDAG &DAG) const override;
 244
 245   bool allowsMisalignedMemoryAccessesImpl(
 246       unsigned Size, unsigned AS, unsigned Align,
 247       MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
 248       bool *IsFast = nullptr) const;
 249
 250   bool allowsMisalignedMemoryAccesses(
 251       EVT VT, unsigned AS, unsigned Align,
 252       MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
 253       bool *IsFast = nullptr) const override;
 254
 255   EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign,
 256                           unsigned SrcAlign, bool IsMemset,
 257                           bool ZeroMemset,
 258                           bool MemcpyStrSrc,
 259                           const AttributeList &FuncAttributes) const override;
 260
 261   bool isMemOpUniform(const SDNode *N) const;
 262   bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const;
 263   bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
 264   bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
 265
 266   TargetLoweringBase::LegalizeTypeAction
 267   getPreferredVectorAction(MVT VT) const override;
 268
 269   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
 270                                         Type *Ty) const override;
 271
 272   bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
 273
 274   bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
 275
 276   bool supportSplitCSR(MachineFunction *MF) const override;
 277   void initializeSplitCSR(MachineBasicBlock *Entry) const override;
 278   void insertCopiesSplitCSR(
 279     MachineBasicBlock *Entry,
 280     const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
 281
 282   SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
 283                                bool isVarArg,
 284                                const SmallVectorImpl<ISD::InputArg> &Ins,
 285                                const SDLoc &DL, SelectionDAG &DAG,
 286                                SmallVectorImpl<SDValue> &InVals) const override;
 287
 288   bool CanLowerReturn(CallingConv::ID CallConv,
 289                       MachineFunction &MF, bool isVarArg,
 290                       const SmallVectorImpl<ISD::OutputArg> &Outs,
 291                       LLVMContext &Context) const override;
 292
 293   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
 294                       const SmallVectorImpl<ISD::OutputArg> &Outs,
 295                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
 296                       SelectionDAG &DAG) const override;
 297
 298   void passSpecialInputs(
 299     CallLoweringInfo &CLI,
 300     CCState &CCInfo,
 301     const SIMachineFunctionInfo &Info,
 302     SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
 303     SmallVectorImpl<SDValue> &MemOpChains,
 304     SDValue Chain) const;
 305
 306   SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
 307                           CallingConv::ID CallConv, bool isVarArg,
 308                           const SmallVectorImpl<ISD::InputArg> &Ins,
 309                           const SDLoc &DL, SelectionDAG &DAG,
 310                           SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
 311                           SDValue ThisVal) const;
 312
 313   bool mayBeEmittedAsTailCall(const CallInst *) const override;
 314
 315   bool isEligibleForTailCallOptimization(
 316     SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
 317     const SmallVectorImpl<ISD::OutputArg> &Outs,
 318     const SmallVectorImpl<SDValue> &OutVals,
 319     const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
 320
 321   SDValue LowerCall(CallLoweringInfo &CLI,
 322                     SmallVectorImpl<SDValue> &InVals) const override;
 323
 324   Register getRegisterByName(const char* RegName, EVT VT,
 325                              const MachineFunction &MF) const override;
 326
 327   MachineBasicBlock *splitKillBlock(MachineInstr &MI,
 328                                     MachineBasicBlock *BB) const;
 329
 330   void bundleInstWithWaitcnt(MachineInstr &MI) const;
 331   MachineBasicBlock *emitGWSMemViolTestLoop(MachineInstr &MI,
 332                                             MachineBasicBlock *BB) const;
 333
 334   MachineBasicBlock *
 335   EmitInstrWithCustomInserter(MachineInstr &MI,
 336                               MachineBasicBlock *BB) const override;
 337
 338   bool hasBitPreservingFPLogic(EVT VT) const override;
 339   bool enableAggressiveFMAFusion(EVT VT) const override;
 340   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
 341                          EVT VT) const override;
 342   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
 343   bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
 344   SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
 345   SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
 346   SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const;
 347   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 348
 349   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
 350                           SelectionDAG &DAG) const override;
 351
 352   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 353   SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override;
 354   void AdjustInstrPostInstrSelection(MachineInstr &MI,
 355                                      SDNode *Node) const override;
 356
 357   SDNode *legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const;
 358
 359   MachineSDNode *wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL,
 360                                 SDValue Ptr) const;
 361   MachineSDNode *buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr,
 362                            uint32_t RsrcDword1, uint64_t RsrcDword2And3) const;
 363   std::pair<unsigned, const TargetRegisterClass *>
 364   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
 365                                StringRef Constraint, MVT VT) const override;
 366   ConstraintType getConstraintType(StringRef Constraint) const override;
 367   SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL,
 368                    SDValue V) const;
 369
 370   void finalizeLowering(MachineFunction &MF) const override;
 371
 372   void computeKnownBitsForFrameIndex(const SDValue Op,
 373                                      KnownBits &Known,
 374                                      const APInt &DemandedElts,
 375                                      const SelectionDAG &DAG,
 376                                      unsigned Depth = 0) const override;
 377
 378   bool isSDNodeSourceOfDivergence(const SDNode *N,
 379     FunctionLoweringInfo *FLI, LegacyDivergenceAnalysis *DA) const override;
 380
 381   bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
 382                        unsigned MaxDepth = 5) const;
 383   bool denormalsEnabledForType(EVT VT) const;
 384
 385   bool isKnownNeverNaNForTargetNode(SDValue Op,
 386                                     const SelectionDAG &DAG,
 387                                     bool SNaN = false,
 388                                     unsigned Depth = 0) const override;
 389   AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
 390
 391   Align getPrefLoopAlignment(MachineLoop *ML) const override;
 392
 393   void allocateHSAUserSGPRs(CCState &CCInfo,
 394                             MachineFunction &MF,
 395                             const SIRegisterInfo &TRI,
 396                             SIMachineFunctionInfo &Info) const;
 397
 398   void allocateSystemSGPRs(CCState &CCInfo,
 399                            MachineFunction &MF,
 400                            SIMachineFunctionInfo &Info,
 401                            CallingConv::ID CallConv,
 402                            bool IsShader) const;
 403
 404   void allocateSpecialEntryInputVGPRs(CCState &CCInfo,
 405                                       MachineFunction &MF,
 406                                       const SIRegisterInfo &TRI,
 407                                       SIMachineFunctionInfo &Info) const;
 408   void allocateSpecialInputSGPRs(
 409     CCState &CCInfo,
 410     MachineFunction &MF,
 411     const SIRegisterInfo &TRI,
 412     SIMachineFunctionInfo &Info) const;
 413
 414   void allocateSpecialInputVGPRs(CCState &CCInfo,
 415                                  MachineFunction &MF,
 416                                  const SIRegisterInfo &TRI,
 417                                  SIMachineFunctionInfo &Info) const;
 418 };
 419
 420 } // End namespace llvm
 421
 422 #endif