1 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// SI DAG Lowering interface definition
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
15 #define LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
17 #include "AMDGPUISelLowering.h"
18 #include "AMDGPUArgumentUsageInfo.h"
19 #include "SIInstrInfo.h"
23 class SITargetLowering final
: public AMDGPUTargetLowering
{
25 const GCNSubtarget
*Subtarget
;
28 MVT
getRegisterTypeForCallingConv(LLVMContext
&Context
,
30 EVT VT
) const override
;
31 unsigned getNumRegistersForCallingConv(LLVMContext
&Context
,
33 EVT VT
) const override
;
35 unsigned getVectorTypeBreakdownForCallingConv(
36 LLVMContext
&Context
, CallingConv::ID CC
, EVT VT
, EVT
&IntermediateVT
,
37 unsigned &NumIntermediates
, MVT
&RegisterVT
) const override
;
40 SDValue
lowerKernArgParameterPtr(SelectionDAG
&DAG
, const SDLoc
&SL
,
41 SDValue Chain
, uint64_t Offset
) const;
42 SDValue
getImplicitArgPtr(SelectionDAG
&DAG
, const SDLoc
&SL
) const;
43 SDValue
lowerKernargMemParameter(SelectionDAG
&DAG
, EVT VT
, EVT MemVT
,
44 const SDLoc
&SL
, SDValue Chain
,
45 uint64_t Offset
, unsigned Align
, bool Signed
,
46 const ISD::InputArg
*Arg
= nullptr) const;
48 SDValue
lowerStackParameter(SelectionDAG
&DAG
, CCValAssign
&VA
,
49 const SDLoc
&SL
, SDValue Chain
,
50 const ISD::InputArg
&Arg
) const;
51 SDValue
getPreloadedValue(SelectionDAG
&DAG
,
52 const SIMachineFunctionInfo
&MFI
,
54 AMDGPUFunctionArgInfo::PreloadedValue
) const;
56 SDValue
LowerGlobalAddress(AMDGPUMachineFunction
*MFI
, SDValue Op
,
57 SelectionDAG
&DAG
) const override
;
58 SDValue
lowerImplicitZextParam(SelectionDAG
&DAG
, SDValue Op
,
59 MVT VT
, unsigned Offset
) const;
60 SDValue
lowerImage(SDValue Op
, const AMDGPU::ImageDimIntrinsicInfo
*Intr
,
61 SelectionDAG
&DAG
) const;
62 SDValue
lowerSBuffer(EVT VT
, SDLoc DL
, SDValue Rsrc
, SDValue Offset
,
63 SDValue GLC
, SDValue DLC
, SelectionDAG
&DAG
) const;
65 SDValue
LowerINTRINSIC_WO_CHAIN(SDValue Op
, SelectionDAG
&DAG
) const;
66 SDValue
LowerINTRINSIC_W_CHAIN(SDValue Op
, SelectionDAG
&DAG
) const;
67 SDValue
LowerINTRINSIC_VOID(SDValue Op
, SelectionDAG
&DAG
) const;
69 // The raw.tbuffer and struct.tbuffer intrinsics have two offset args: offset
70 // (the offset that is included in bounds checking and swizzling, to be split
71 // between the instruction's voffset and immoffset fields) and soffset (the
72 // offset that is excluded from bounds checking and swizzling, to go in the
73 // instruction's soffset field). This function takes the first kind of
74 // offset and figures out how to split it between voffset and immoffset.
75 std::pair
<SDValue
, SDValue
> splitBufferOffsets(SDValue Offset
,
76 SelectionDAG
&DAG
) const;
78 SDValue
widenLoad(LoadSDNode
*Ld
, DAGCombinerInfo
&DCI
) const;
79 SDValue
LowerLOAD(SDValue Op
, SelectionDAG
&DAG
) const;
80 SDValue
LowerSELECT(SDValue Op
, SelectionDAG
&DAG
) const;
81 SDValue
lowerFastUnsafeFDIV(SDValue Op
, SelectionDAG
&DAG
) const;
82 SDValue
lowerFDIV_FAST(SDValue Op
, SelectionDAG
&DAG
) const;
83 SDValue
LowerFDIV16(SDValue Op
, SelectionDAG
&DAG
) const;
84 SDValue
LowerFDIV32(SDValue Op
, SelectionDAG
&DAG
) const;
85 SDValue
LowerFDIV64(SDValue Op
, SelectionDAG
&DAG
) const;
86 SDValue
LowerFDIV(SDValue Op
, SelectionDAG
&DAG
) const;
87 SDValue
LowerINT_TO_FP(SDValue Op
, SelectionDAG
&DAG
, bool Signed
) const;
88 SDValue
LowerSTORE(SDValue Op
, SelectionDAG
&DAG
) const;
89 SDValue
LowerTrig(SDValue Op
, SelectionDAG
&DAG
) const;
90 SDValue
LowerATOMIC_CMP_SWAP(SDValue Op
, SelectionDAG
&DAG
) const;
91 SDValue
LowerBRCOND(SDValue Op
, SelectionDAG
&DAG
) const;
92 SDValue
LowerRETURNADDR(SDValue Op
, SelectionDAG
&DAG
) const;
93 SDValue
adjustLoadValueType(unsigned Opcode
, MemSDNode
*M
,
94 SelectionDAG
&DAG
, ArrayRef
<SDValue
> Ops
,
95 bool IsIntrinsic
= false) const;
97 SDValue
lowerIntrinsicLoad(MemSDNode
*M
, bool IsFormat
, SelectionDAG
&DAG
,
98 ArrayRef
<SDValue
> Ops
) const;
100 // Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
102 SDValue
getMemIntrinsicNode(unsigned Opcode
, const SDLoc
&DL
, SDVTList VTList
,
103 ArrayRef
<SDValue
> Ops
, EVT MemVT
,
104 MachineMemOperand
*MMO
, SelectionDAG
&DAG
) const;
106 SDValue
handleD16VData(SDValue VData
, SelectionDAG
&DAG
) const;
108 /// Converts \p Op, which must be of floating point type, to the
109 /// floating point type \p VT, by either extending or truncating it.
110 SDValue
getFPExtOrFPTrunc(SelectionDAG
&DAG
,
115 SDValue
convertArgType(
116 SelectionDAG
&DAG
, EVT VT
, EVT MemVT
, const SDLoc
&SL
, SDValue Val
,
117 bool Signed
, const ISD::InputArg
*Arg
= nullptr) const;
119 /// Custom lowering for ISD::FP_ROUND for MVT::f16.
120 SDValue
lowerFP_ROUND(SDValue Op
, SelectionDAG
&DAG
) const;
121 SDValue
lowerFMINNUM_FMAXNUM(SDValue Op
, SelectionDAG
&DAG
) const;
123 SDValue
getSegmentAperture(unsigned AS
, const SDLoc
&DL
,
124 SelectionDAG
&DAG
) const;
126 SDValue
lowerADDRSPACECAST(SDValue Op
, SelectionDAG
&DAG
) const;
127 SDValue
lowerINSERT_SUBVECTOR(SDValue Op
, SelectionDAG
&DAG
) const;
128 SDValue
lowerINSERT_VECTOR_ELT(SDValue Op
, SelectionDAG
&DAG
) const;
129 SDValue
lowerEXTRACT_VECTOR_ELT(SDValue Op
, SelectionDAG
&DAG
) const;
130 SDValue
lowerVECTOR_SHUFFLE(SDValue Op
, SelectionDAG
&DAG
) const;
131 SDValue
lowerBUILD_VECTOR(SDValue Op
, SelectionDAG
&DAG
) const;
132 SDValue
lowerTRAP(SDValue Op
, SelectionDAG
&DAG
) const;
133 SDValue
lowerDEBUGTRAP(SDValue Op
, SelectionDAG
&DAG
) const;
135 SDNode
*adjustWritemask(MachineSDNode
*&N
, SelectionDAG
&DAG
) const;
137 SDValue
performUCharToFloatCombine(SDNode
*N
,
138 DAGCombinerInfo
&DCI
) const;
139 SDValue
performSHLPtrCombine(SDNode
*N
,
142 DAGCombinerInfo
&DCI
) const;
144 SDValue
performMemSDNodeCombine(MemSDNode
*N
, DAGCombinerInfo
&DCI
) const;
146 SDValue
splitBinaryBitConstantOp(DAGCombinerInfo
&DCI
, const SDLoc
&SL
,
147 unsigned Opc
, SDValue LHS
,
148 const ConstantSDNode
*CRHS
) const;
150 SDValue
performAndCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
151 SDValue
performOrCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
152 SDValue
performXorCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
153 SDValue
performZeroExtendCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
154 SDValue
performSignExtendInRegCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
155 SDValue
performClassCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
156 SDValue
getCanonicalConstantFP(SelectionDAG
&DAG
, const SDLoc
&SL
, EVT VT
,
157 const APFloat
&C
) const;
158 SDValue
performFCanonicalizeCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
160 SDValue
performFPMed3ImmCombine(SelectionDAG
&DAG
, const SDLoc
&SL
,
161 SDValue Op0
, SDValue Op1
) const;
162 SDValue
performIntMed3ImmCombine(SelectionDAG
&DAG
, const SDLoc
&SL
,
163 SDValue Op0
, SDValue Op1
, bool Signed
) const;
164 SDValue
performMinMaxCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
165 SDValue
performFMed3Combine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
166 SDValue
performCvtPkRTZCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
167 SDValue
performExtractVectorEltCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
168 SDValue
performInsertVectorEltCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
170 SDValue
reassociateScalarOps(SDNode
*N
, SelectionDAG
&DAG
) const;
171 unsigned getFusedOpcode(const SelectionDAG
&DAG
,
172 const SDNode
*N0
, const SDNode
*N1
) const;
173 SDValue
performAddCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
174 SDValue
performAddCarrySubCarryCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
175 SDValue
performSubCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
176 SDValue
performFAddCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
177 SDValue
performFSubCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
178 SDValue
performFMACombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
179 SDValue
performSetCCCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
180 SDValue
performCvtF32UByteNCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
181 SDValue
performClampCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
182 SDValue
performRcpCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const;
184 bool isLegalFlatAddressingMode(const AddrMode
&AM
) const;
185 bool isLegalMUBUFAddressingMode(const AddrMode
&AM
) const;
187 unsigned isCFIntrinsic(const SDNode
*Intr
) const;
190 /// \returns True if fixup needs to be emitted for given global value \p GV,
192 bool shouldEmitFixup(const GlobalValue
*GV
) const;
194 /// \returns True if GOT relocation needs to be emitted for given global value
195 /// \p GV, false otherwise.
196 bool shouldEmitGOTReloc(const GlobalValue
*GV
) const;
198 /// \returns True if PC-relative relocation needs to be emitted for given
199 /// global value \p GV, false otherwise.
200 bool shouldEmitPCReloc(const GlobalValue
*GV
) const;
203 // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
204 // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
205 // pointed to by Offsets.
206 /// \returns 0 If there is a non-constant offset or if the offset is 0.
207 /// Otherwise returns the constant offset.
208 unsigned setBufferOffsets(SDValue CombinedOffset
, SelectionDAG
&DAG
,
209 SDValue
*Offsets
, unsigned Align
= 4) const;
211 // Handle 8 bit and 16 bit buffer loads
212 SDValue
handleByteShortBufferLoads(SelectionDAG
&DAG
, EVT LoadVT
, SDLoc DL
,
213 ArrayRef
<SDValue
> Ops
, MemSDNode
*M
) const;
215 // Handle 8 bit and 16 bit buffer stores
216 SDValue
handleByteShortBufferStores(SelectionDAG
&DAG
, EVT VDataType
,
217 SDLoc DL
, SDValue Ops
[],
221 SITargetLowering(const TargetMachine
&tm
, const GCNSubtarget
&STI
);
223 const GCNSubtarget
*getSubtarget() const;
225 bool isFPExtFoldable(unsigned Opcode
, EVT DestVT
, EVT SrcVT
) const override
;
227 bool isShuffleMaskLegal(ArrayRef
<int> /*Mask*/, EVT
/*VT*/) const override
;
229 bool getTgtMemIntrinsic(IntrinsicInfo
&, const CallInst
&,
231 unsigned IntrinsicID
) const override
;
233 bool getAddrModeArguments(IntrinsicInst
* /*I*/,
234 SmallVectorImpl
<Value
*> &/*Ops*/,
235 Type
*&/*AccessTy*/) const override
;
237 bool isLegalGlobalAddressingMode(const AddrMode
&AM
) const;
238 bool isLegalAddressingMode(const DataLayout
&DL
, const AddrMode
&AM
, Type
*Ty
,
240 Instruction
*I
= nullptr) const override
;
242 bool canMergeStoresTo(unsigned AS
, EVT MemVT
,
243 const SelectionDAG
&DAG
) const override
;
245 bool allowsMisalignedMemoryAccessesImpl(
246 unsigned Size
, unsigned AS
, unsigned Align
,
247 MachineMemOperand::Flags Flags
= MachineMemOperand::MONone
,
248 bool *IsFast
= nullptr) const;
250 bool allowsMisalignedMemoryAccesses(
251 EVT VT
, unsigned AS
, unsigned Align
,
252 MachineMemOperand::Flags Flags
= MachineMemOperand::MONone
,
253 bool *IsFast
= nullptr) const override
;
255 EVT
getOptimalMemOpType(uint64_t Size
, unsigned DstAlign
,
256 unsigned SrcAlign
, bool IsMemset
,
259 const AttributeList
&FuncAttributes
) const override
;
261 bool isMemOpUniform(const SDNode
*N
) const;
262 bool isMemOpHasNoClobberedMemOperand(const SDNode
*N
) const;
263 bool isNoopAddrSpaceCast(unsigned SrcAS
, unsigned DestAS
) const override
;
264 bool isFreeAddrSpaceCast(unsigned SrcAS
, unsigned DestAS
) const override
;
266 TargetLoweringBase::LegalizeTypeAction
267 getPreferredVectorAction(MVT VT
) const override
;
269 bool shouldConvertConstantLoadToIntImm(const APInt
&Imm
,
270 Type
*Ty
) const override
;
272 bool isTypeDesirableForOp(unsigned Op
, EVT VT
) const override
;
274 bool isOffsetFoldingLegal(const GlobalAddressSDNode
*GA
) const override
;
276 bool supportSplitCSR(MachineFunction
*MF
) const override
;
277 void initializeSplitCSR(MachineBasicBlock
*Entry
) const override
;
278 void insertCopiesSplitCSR(
279 MachineBasicBlock
*Entry
,
280 const SmallVectorImpl
<MachineBasicBlock
*> &Exits
) const override
;
282 SDValue
LowerFormalArguments(SDValue Chain
, CallingConv::ID CallConv
,
284 const SmallVectorImpl
<ISD::InputArg
> &Ins
,
285 const SDLoc
&DL
, SelectionDAG
&DAG
,
286 SmallVectorImpl
<SDValue
> &InVals
) const override
;
288 bool CanLowerReturn(CallingConv::ID CallConv
,
289 MachineFunction
&MF
, bool isVarArg
,
290 const SmallVectorImpl
<ISD::OutputArg
> &Outs
,
291 LLVMContext
&Context
) const override
;
293 SDValue
LowerReturn(SDValue Chain
, CallingConv::ID CallConv
, bool IsVarArg
,
294 const SmallVectorImpl
<ISD::OutputArg
> &Outs
,
295 const SmallVectorImpl
<SDValue
> &OutVals
, const SDLoc
&DL
,
296 SelectionDAG
&DAG
) const override
;
298 void passSpecialInputs(
299 CallLoweringInfo
&CLI
,
301 const SIMachineFunctionInfo
&Info
,
302 SmallVectorImpl
<std::pair
<unsigned, SDValue
>> &RegsToPass
,
303 SmallVectorImpl
<SDValue
> &MemOpChains
,
304 SDValue Chain
) const;
306 SDValue
LowerCallResult(SDValue Chain
, SDValue InFlag
,
307 CallingConv::ID CallConv
, bool isVarArg
,
308 const SmallVectorImpl
<ISD::InputArg
> &Ins
,
309 const SDLoc
&DL
, SelectionDAG
&DAG
,
310 SmallVectorImpl
<SDValue
> &InVals
, bool isThisReturn
,
311 SDValue ThisVal
) const;
313 bool mayBeEmittedAsTailCall(const CallInst
*) const override
;
315 bool isEligibleForTailCallOptimization(
316 SDValue Callee
, CallingConv::ID CalleeCC
, bool isVarArg
,
317 const SmallVectorImpl
<ISD::OutputArg
> &Outs
,
318 const SmallVectorImpl
<SDValue
> &OutVals
,
319 const SmallVectorImpl
<ISD::InputArg
> &Ins
, SelectionDAG
&DAG
) const;
321 SDValue
LowerCall(CallLoweringInfo
&CLI
,
322 SmallVectorImpl
<SDValue
> &InVals
) const override
;
324 Register
getRegisterByName(const char* RegName
, EVT VT
,
325 const MachineFunction
&MF
) const override
;
327 MachineBasicBlock
*splitKillBlock(MachineInstr
&MI
,
328 MachineBasicBlock
*BB
) const;
330 void bundleInstWithWaitcnt(MachineInstr
&MI
) const;
331 MachineBasicBlock
*emitGWSMemViolTestLoop(MachineInstr
&MI
,
332 MachineBasicBlock
*BB
) const;
335 EmitInstrWithCustomInserter(MachineInstr
&MI
,
336 MachineBasicBlock
*BB
) const override
;
338 bool hasBitPreservingFPLogic(EVT VT
) const override
;
339 bool enableAggressiveFMAFusion(EVT VT
) const override
;
340 EVT
getSetCCResultType(const DataLayout
&DL
, LLVMContext
&Context
,
341 EVT VT
) const override
;
342 MVT
getScalarShiftAmountTy(const DataLayout
&, EVT
) const override
;
343 bool isFMAFasterThanFMulAndFAdd(EVT VT
) const override
;
344 SDValue
splitUnaryVectorOp(SDValue Op
, SelectionDAG
&DAG
) const;
345 SDValue
splitBinaryVectorOp(SDValue Op
, SelectionDAG
&DAG
) const;
346 SDValue
splitTernaryVectorOp(SDValue Op
, SelectionDAG
&DAG
) const;
347 SDValue
LowerOperation(SDValue Op
, SelectionDAG
&DAG
) const override
;
349 void ReplaceNodeResults(SDNode
*N
, SmallVectorImpl
<SDValue
> &Results
,
350 SelectionDAG
&DAG
) const override
;
352 SDValue
PerformDAGCombine(SDNode
*N
, DAGCombinerInfo
&DCI
) const override
;
353 SDNode
*PostISelFolding(MachineSDNode
*N
, SelectionDAG
&DAG
) const override
;
354 void AdjustInstrPostInstrSelection(MachineInstr
&MI
,
355 SDNode
*Node
) const override
;
357 SDNode
*legalizeTargetIndependentNode(SDNode
*Node
, SelectionDAG
&DAG
) const;
359 MachineSDNode
*wrapAddr64Rsrc(SelectionDAG
&DAG
, const SDLoc
&DL
,
361 MachineSDNode
*buildRSRC(SelectionDAG
&DAG
, const SDLoc
&DL
, SDValue Ptr
,
362 uint32_t RsrcDword1
, uint64_t RsrcDword2And3
) const;
363 std::pair
<unsigned, const TargetRegisterClass
*>
364 getRegForInlineAsmConstraint(const TargetRegisterInfo
*TRI
,
365 StringRef Constraint
, MVT VT
) const override
;
366 ConstraintType
getConstraintType(StringRef Constraint
) const override
;
367 SDValue
copyToM0(SelectionDAG
&DAG
, SDValue Chain
, const SDLoc
&DL
,
370 void finalizeLowering(MachineFunction
&MF
) const override
;
372 void computeKnownBitsForFrameIndex(const SDValue Op
,
374 const APInt
&DemandedElts
,
375 const SelectionDAG
&DAG
,
376 unsigned Depth
= 0) const override
;
378 bool isSDNodeSourceOfDivergence(const SDNode
*N
,
379 FunctionLoweringInfo
*FLI
, LegacyDivergenceAnalysis
*DA
) const override
;
381 bool isCanonicalized(SelectionDAG
&DAG
, SDValue Op
,
382 unsigned MaxDepth
= 5) const;
383 bool denormalsEnabledForType(EVT VT
) const;
385 bool isKnownNeverNaNForTargetNode(SDValue Op
,
386 const SelectionDAG
&DAG
,
388 unsigned Depth
= 0) const override
;
389 AtomicExpansionKind
shouldExpandAtomicRMWInIR(AtomicRMWInst
*) const override
;
391 virtual const TargetRegisterClass
*
392 getRegClassFor(MVT VT
, bool isDivergent
) const override
;
393 virtual bool requiresUniformRegister(MachineFunction
&MF
,
394 const Value
*V
) const override
;
395 Align
getPrefLoopAlignment(MachineLoop
*ML
) const override
;
397 void allocateHSAUserSGPRs(CCState
&CCInfo
,
399 const SIRegisterInfo
&TRI
,
400 SIMachineFunctionInfo
&Info
) const;
402 void allocateSystemSGPRs(CCState
&CCInfo
,
404 SIMachineFunctionInfo
&Info
,
405 CallingConv::ID CallConv
,
406 bool IsShader
) const;
408 void allocateSpecialEntryInputVGPRs(CCState
&CCInfo
,
410 const SIRegisterInfo
&TRI
,
411 SIMachineFunctionInfo
&Info
) const;
412 void allocateSpecialInputSGPRs(
415 const SIRegisterInfo
&TRI
,
416 SIMachineFunctionInfo
&Info
) const;
418 void allocateSpecialInputVGPRs(CCState
&CCInfo
,
420 const SIRegisterInfo
&TRI
,
421 SIMachineFunctionInfo
&Info
) const;
424 } // End namespace llvm