1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// SI Implementation of TargetInstrInfo.
12 //===----------------------------------------------------------------------===//
14 #include "SIInstrInfo.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/iterator_range.h"
28 #include "llvm/Analysis/AliasAnalysis.h"
29 #include "llvm/Analysis/MemoryLocation.h"
30 #include "llvm/Analysis/ValueTracking.h"
31 #include "llvm/CodeGen/MachineBasicBlock.h"
32 #include "llvm/CodeGen/MachineDominators.h"
33 #include "llvm/CodeGen/MachineFrameInfo.h"
34 #include "llvm/CodeGen/MachineFunction.h"
35 #include "llvm/CodeGen/MachineInstr.h"
36 #include "llvm/CodeGen/MachineInstrBuilder.h"
37 #include "llvm/CodeGen/MachineInstrBundle.h"
38 #include "llvm/CodeGen/MachineMemOperand.h"
39 #include "llvm/CodeGen/MachineOperand.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/RegisterScavenging.h"
42 #include "llvm/CodeGen/ScheduleDAG.h"
43 #include "llvm/CodeGen/SelectionDAGNodes.h"
44 #include "llvm/CodeGen/TargetOpcodes.h"
45 #include "llvm/CodeGen/TargetRegisterInfo.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/CommandLine.h"
54 #include "llvm/Support/Compiler.h"
55 #include "llvm/Support/ErrorHandling.h"
56 #include "llvm/Support/MachineValueType.h"
57 #include "llvm/Support/MathExtras.h"
58 #include "llvm/Target/TargetMachine.h"
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
82 static cl::opt
<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden
, cl::init(16),
84 cl::desc("Restrict range of branch instructions (DEBUG)"));
86 SIInstrInfo::SIInstrInfo(const GCNSubtarget
&ST
)
87 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP
, AMDGPU::ADJCALLSTACKDOWN
),
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
94 static unsigned getNumOperandsNoGlue(SDNode
*Node
) {
95 unsigned N
= Node
->getNumOperands();
96 while (N
&& Node
->getOperand(N
- 1).getValueType() == MVT::Glue
)
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode
*N0
, SDNode
* N1
, unsigned OpName
) {
104 unsigned Opc0
= N0
->getMachineOpcode();
105 unsigned Opc1
= N1
->getMachineOpcode();
107 int Op0Idx
= AMDGPU::getNamedOperandIdx(Opc0
, OpName
);
108 int Op1Idx
= AMDGPU::getNamedOperandIdx(Opc1
, OpName
);
110 if (Op0Idx
== -1 && Op1Idx
== -1)
114 if ((Op0Idx
== -1 && Op1Idx
!= -1) ||
115 (Op1Idx
== -1 && Op0Idx
!= -1))
118 // getNamedOperandIdx returns the index for the MachineInstr's operands,
119 // which includes the result as the first operand. We are indexing into the
120 // MachineSDNode's operands, so we need to skip the result operand to get
125 return N0
->getOperand(Op0Idx
) == N1
->getOperand(Op1Idx
);
128 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr
&MI
,
129 AliasAnalysis
*AA
) const {
130 // TODO: The generic check fails for VALU instructions that should be
131 // rematerializable due to implicit reads of exec. We really want all of the
132 // generic logic for this except for this.
133 switch (MI
.getOpcode()) {
134 case AMDGPU::V_MOV_B32_e32
:
135 case AMDGPU::V_MOV_B32_e64
:
136 case AMDGPU::V_MOV_B64_PSEUDO
:
137 // No implicit operands.
138 return MI
.getNumOperands() == MI
.getDesc().getNumOperands();
144 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode
*Load0
, SDNode
*Load1
,
146 int64_t &Offset1
) const {
147 if (!Load0
->isMachineOpcode() || !Load1
->isMachineOpcode())
150 unsigned Opc0
= Load0
->getMachineOpcode();
151 unsigned Opc1
= Load1
->getMachineOpcode();
153 // Make sure both are actually loads.
154 if (!get(Opc0
).mayLoad() || !get(Opc1
).mayLoad())
157 if (isDS(Opc0
) && isDS(Opc1
)) {
159 // FIXME: Handle this case:
160 if (getNumOperandsNoGlue(Load0
) != getNumOperandsNoGlue(Load1
))
164 if (Load0
->getOperand(0) != Load1
->getOperand(0))
167 // Skip read2 / write2 variants for simplicity.
168 // TODO: We should report true if the used offsets are adjacent (excluded
170 int Offset0Idx
= AMDGPU::getNamedOperandIdx(Opc0
, AMDGPU::OpName::offset
);
171 int Offset1Idx
= AMDGPU::getNamedOperandIdx(Opc1
, AMDGPU::OpName::offset
);
172 if (Offset0Idx
== -1 || Offset1Idx
== -1)
175 // XXX - be careful of datalesss loads
176 // getNamedOperandIdx returns the index for MachineInstrs. Since they
177 // include the output in the operand list, but SDNodes don't, we need to
178 // subtract the index by one.
179 Offset0Idx
-= get(Opc0
).NumDefs
;
180 Offset1Idx
-= get(Opc1
).NumDefs
;
181 Offset0
= cast
<ConstantSDNode
>(Load0
->getOperand(Offset0Idx
))->getZExtValue();
182 Offset1
= cast
<ConstantSDNode
>(Load1
->getOperand(Offset1Idx
))->getZExtValue();
186 if (isSMRD(Opc0
) && isSMRD(Opc1
)) {
187 // Skip time and cache invalidation instructions.
188 if (AMDGPU::getNamedOperandIdx(Opc0
, AMDGPU::OpName::sbase
) == -1 ||
189 AMDGPU::getNamedOperandIdx(Opc1
, AMDGPU::OpName::sbase
) == -1)
192 assert(getNumOperandsNoGlue(Load0
) == getNumOperandsNoGlue(Load1
));
195 if (Load0
->getOperand(0) != Load1
->getOperand(0))
198 const ConstantSDNode
*Load0Offset
=
199 dyn_cast
<ConstantSDNode
>(Load0
->getOperand(1));
200 const ConstantSDNode
*Load1Offset
=
201 dyn_cast
<ConstantSDNode
>(Load1
->getOperand(1));
203 if (!Load0Offset
|| !Load1Offset
)
206 Offset0
= Load0Offset
->getZExtValue();
207 Offset1
= Load1Offset
->getZExtValue();
211 // MUBUF and MTBUF can access the same addresses.
212 if ((isMUBUF(Opc0
) || isMTBUF(Opc0
)) && (isMUBUF(Opc1
) || isMTBUF(Opc1
))) {
214 // MUBUF and MTBUF have vaddr at different indices.
215 if (!nodesHaveSameOperandValue(Load0
, Load1
, AMDGPU::OpName::soffset
) ||
216 !nodesHaveSameOperandValue(Load0
, Load1
, AMDGPU::OpName::vaddr
) ||
217 !nodesHaveSameOperandValue(Load0
, Load1
, AMDGPU::OpName::srsrc
))
220 int OffIdx0
= AMDGPU::getNamedOperandIdx(Opc0
, AMDGPU::OpName::offset
);
221 int OffIdx1
= AMDGPU::getNamedOperandIdx(Opc1
, AMDGPU::OpName::offset
);
223 if (OffIdx0
== -1 || OffIdx1
== -1)
226 // getNamedOperandIdx returns the index for MachineInstrs. Since they
227 // include the output in the operand list, but SDNodes don't, we need to
228 // subtract the index by one.
229 OffIdx0
-= get(Opc0
).NumDefs
;
230 OffIdx1
-= get(Opc1
).NumDefs
;
232 SDValue Off0
= Load0
->getOperand(OffIdx0
);
233 SDValue Off1
= Load1
->getOperand(OffIdx1
);
235 // The offset might be a FrameIndexSDNode.
236 if (!isa
<ConstantSDNode
>(Off0
) || !isa
<ConstantSDNode
>(Off1
))
239 Offset0
= cast
<ConstantSDNode
>(Off0
)->getZExtValue();
240 Offset1
= cast
<ConstantSDNode
>(Off1
)->getZExtValue();
247 static bool isStride64(unsigned Opc
) {
249 case AMDGPU::DS_READ2ST64_B32
:
250 case AMDGPU::DS_READ2ST64_B64
:
251 case AMDGPU::DS_WRITE2ST64_B32
:
252 case AMDGPU::DS_WRITE2ST64_B64
:
259 bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr
&LdSt
,
260 const MachineOperand
*&BaseOp
,
262 const TargetRegisterInfo
*TRI
) const {
263 unsigned Opc
= LdSt
.getOpcode();
266 const MachineOperand
*OffsetImm
=
267 getNamedOperand(LdSt
, AMDGPU::OpName::offset
);
269 // Normal, single offset LDS instruction.
270 BaseOp
= getNamedOperand(LdSt
, AMDGPU::OpName::addr
);
271 // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
276 Offset
= OffsetImm
->getImm();
277 assert(BaseOp
->isReg() && "getMemOperandWithOffset only supports base "
278 "operands of type register.");
282 // The 2 offset instructions use offset0 and offset1 instead. We can treat
283 // these as a load with a single offset if the 2 offsets are consecutive. We
284 // will use this for some partially aligned loads.
285 const MachineOperand
*Offset0Imm
=
286 getNamedOperand(LdSt
, AMDGPU::OpName::offset0
);
287 const MachineOperand
*Offset1Imm
=
288 getNamedOperand(LdSt
, AMDGPU::OpName::offset1
);
290 uint8_t Offset0
= Offset0Imm
->getImm();
291 uint8_t Offset1
= Offset1Imm
->getImm();
293 if (Offset1
> Offset0
&& Offset1
- Offset0
== 1) {
294 // Each of these offsets is in element sized units, so we need to convert
295 // to bytes of the individual reads.
299 EltSize
= TRI
->getRegSizeInBits(*getOpRegClass(LdSt
, 0)) / 16;
301 assert(LdSt
.mayStore());
302 int Data0Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::data0
);
303 EltSize
= TRI
->getRegSizeInBits(*getOpRegClass(LdSt
, Data0Idx
)) / 8;
309 BaseOp
= getNamedOperand(LdSt
, AMDGPU::OpName::addr
);
310 Offset
= EltSize
* Offset0
;
311 assert(BaseOp
->isReg() && "getMemOperandWithOffset only supports base "
312 "operands of type register.");
319 if (isMUBUF(LdSt
) || isMTBUF(LdSt
)) {
320 const MachineOperand
*SOffset
= getNamedOperand(LdSt
, AMDGPU::OpName::soffset
);
321 if (SOffset
&& SOffset
->isReg()) {
322 // We can only handle this if it's a stack access, as any other resource
323 // would require reporting multiple base registers.
324 const MachineOperand
*AddrReg
= getNamedOperand(LdSt
, AMDGPU::OpName::vaddr
);
325 if (AddrReg
&& !AddrReg
->isFI())
328 const MachineOperand
*RSrc
= getNamedOperand(LdSt
, AMDGPU::OpName::srsrc
);
329 const SIMachineFunctionInfo
*MFI
330 = LdSt
.getParent()->getParent()->getInfo
<SIMachineFunctionInfo
>();
331 if (RSrc
->getReg() != MFI
->getScratchRSrcReg())
334 const MachineOperand
*OffsetImm
=
335 getNamedOperand(LdSt
, AMDGPU::OpName::offset
);
337 Offset
= OffsetImm
->getImm();
341 const MachineOperand
*AddrReg
= getNamedOperand(LdSt
, AMDGPU::OpName::vaddr
);
345 const MachineOperand
*OffsetImm
=
346 getNamedOperand(LdSt
, AMDGPU::OpName::offset
);
348 Offset
= OffsetImm
->getImm();
350 if (SOffset
) // soffset can be an inline immediate.
351 Offset
+= SOffset
->getImm();
353 assert(BaseOp
->isReg() && "getMemOperandWithOffset only supports base "
354 "operands of type register.");
359 const MachineOperand
*OffsetImm
=
360 getNamedOperand(LdSt
, AMDGPU::OpName::offset
);
364 const MachineOperand
*SBaseReg
= getNamedOperand(LdSt
, AMDGPU::OpName::sbase
);
366 Offset
= OffsetImm
->getImm();
367 assert(BaseOp
->isReg() && "getMemOperandWithOffset only supports base "
368 "operands of type register.");
373 const MachineOperand
*VAddr
= getNamedOperand(LdSt
, AMDGPU::OpName::vaddr
);
375 // Can't analyze 2 offsets.
376 if (getNamedOperand(LdSt
, AMDGPU::OpName::saddr
))
381 // scratch instructions have either vaddr or saddr.
382 BaseOp
= getNamedOperand(LdSt
, AMDGPU::OpName::saddr
);
385 Offset
= getNamedOperand(LdSt
, AMDGPU::OpName::offset
)->getImm();
386 assert(BaseOp
->isReg() && "getMemOperandWithOffset only supports base "
387 "operands of type register.");
394 static bool memOpsHaveSameBasePtr(const MachineInstr
&MI1
,
395 const MachineOperand
&BaseOp1
,
396 const MachineInstr
&MI2
,
397 const MachineOperand
&BaseOp2
) {
398 // Support only base operands with base registers.
399 // Note: this could be extended to support FI operands.
400 if (!BaseOp1
.isReg() || !BaseOp2
.isReg())
403 if (BaseOp1
.isIdenticalTo(BaseOp2
))
406 if (!MI1
.hasOneMemOperand() || !MI2
.hasOneMemOperand())
409 auto MO1
= *MI1
.memoperands_begin();
410 auto MO2
= *MI2
.memoperands_begin();
411 if (MO1
->getAddrSpace() != MO2
->getAddrSpace())
414 auto Base1
= MO1
->getValue();
415 auto Base2
= MO2
->getValue();
416 if (!Base1
|| !Base2
)
418 const MachineFunction
&MF
= *MI1
.getParent()->getParent();
419 const DataLayout
&DL
= MF
.getFunction().getParent()->getDataLayout();
420 Base1
= GetUnderlyingObject(Base1
, DL
);
421 Base2
= GetUnderlyingObject(Base1
, DL
);
423 if (isa
<UndefValue
>(Base1
) || isa
<UndefValue
>(Base2
))
426 return Base1
== Base2
;
429 bool SIInstrInfo::shouldClusterMemOps(const MachineOperand
&BaseOp1
,
430 const MachineOperand
&BaseOp2
,
431 unsigned NumLoads
) const {
432 const MachineInstr
&FirstLdSt
= *BaseOp1
.getParent();
433 const MachineInstr
&SecondLdSt
= *BaseOp2
.getParent();
435 if (!memOpsHaveSameBasePtr(FirstLdSt
, BaseOp1
, SecondLdSt
, BaseOp2
))
438 const MachineOperand
*FirstDst
= nullptr;
439 const MachineOperand
*SecondDst
= nullptr;
441 if ((isMUBUF(FirstLdSt
) && isMUBUF(SecondLdSt
)) ||
442 (isMTBUF(FirstLdSt
) && isMTBUF(SecondLdSt
)) ||
443 (isFLAT(FirstLdSt
) && isFLAT(SecondLdSt
))) {
444 const unsigned MaxGlobalLoadCluster
= 6;
445 if (NumLoads
> MaxGlobalLoadCluster
)
448 FirstDst
= getNamedOperand(FirstLdSt
, AMDGPU::OpName::vdata
);
450 FirstDst
= getNamedOperand(FirstLdSt
, AMDGPU::OpName::vdst
);
451 SecondDst
= getNamedOperand(SecondLdSt
, AMDGPU::OpName::vdata
);
453 SecondDst
= getNamedOperand(SecondLdSt
, AMDGPU::OpName::vdst
);
454 } else if (isSMRD(FirstLdSt
) && isSMRD(SecondLdSt
)) {
455 FirstDst
= getNamedOperand(FirstLdSt
, AMDGPU::OpName::sdst
);
456 SecondDst
= getNamedOperand(SecondLdSt
, AMDGPU::OpName::sdst
);
457 } else if (isDS(FirstLdSt
) && isDS(SecondLdSt
)) {
458 FirstDst
= getNamedOperand(FirstLdSt
, AMDGPU::OpName::vdst
);
459 SecondDst
= getNamedOperand(SecondLdSt
, AMDGPU::OpName::vdst
);
462 if (!FirstDst
|| !SecondDst
)
465 // Try to limit clustering based on the total number of bytes loaded
466 // rather than the number of instructions. This is done to help reduce
467 // register pressure. The method used is somewhat inexact, though,
468 // because it assumes that all loads in the cluster will load the
469 // same number of bytes as FirstLdSt.
471 // The unit of this value is bytes.
472 // FIXME: This needs finer tuning.
473 unsigned LoadClusterThreshold
= 16;
475 const MachineRegisterInfo
&MRI
=
476 FirstLdSt
.getParent()->getParent()->getRegInfo();
478 const Register Reg
= FirstDst
->getReg();
480 const TargetRegisterClass
*DstRC
= Register::isVirtualRegister(Reg
)
481 ? MRI
.getRegClass(Reg
)
482 : RI
.getPhysRegClass(Reg
);
484 return (NumLoads
* (RI
.getRegSizeInBits(*DstRC
) / 8)) <= LoadClusterThreshold
;
487 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
488 // the first 16 loads will be interleaved with the stores, and the next 16 will
489 // be clustered as expected. It should really split into 2 16 store batches.
491 // Loads are clustered until this returns false, rather than trying to schedule
492 // groups of stores. This also means we have to deal with saying different
493 // address space loads should be clustered, and ones which might cause bank
496 // This might be deprecated so it might not be worth that much effort to fix.
497 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode
*Load0
, SDNode
*Load1
,
498 int64_t Offset0
, int64_t Offset1
,
499 unsigned NumLoads
) const {
500 assert(Offset1
> Offset0
&&
501 "Second offset should be larger than first offset!");
502 // If we have less than 16 loads in a row, and the offsets are within 64
503 // bytes, then schedule together.
505 // A cacheline is 64 bytes (for global memory).
506 return (NumLoads
<= 16 && (Offset1
- Offset0
) < 64);
509 static void reportIllegalCopy(const SIInstrInfo
*TII
, MachineBasicBlock
&MBB
,
510 MachineBasicBlock::iterator MI
,
511 const DebugLoc
&DL
, unsigned DestReg
,
512 unsigned SrcReg
, bool KillSrc
) {
513 MachineFunction
*MF
= MBB
.getParent();
514 DiagnosticInfoUnsupported
IllegalCopy(MF
->getFunction(),
515 "illegal SGPR to VGPR copy",
517 LLVMContext
&C
= MF
->getFunction().getContext();
518 C
.diagnose(IllegalCopy
);
520 BuildMI(MBB
, MI
, DL
, TII
->get(AMDGPU::SI_ILLEGAL_COPY
), DestReg
)
521 .addReg(SrcReg
, getKillRegState(KillSrc
));
524 void SIInstrInfo::copyPhysReg(MachineBasicBlock
&MBB
,
525 MachineBasicBlock::iterator MI
,
526 const DebugLoc
&DL
, unsigned DestReg
,
527 unsigned SrcReg
, bool KillSrc
) const {
528 const TargetRegisterClass
*RC
= RI
.getPhysRegClass(DestReg
);
530 if (RC
== &AMDGPU::VGPR_32RegClass
) {
531 assert(AMDGPU::VGPR_32RegClass
.contains(SrcReg
) ||
532 AMDGPU::SReg_32RegClass
.contains(SrcReg
) ||
533 AMDGPU::AGPR_32RegClass
.contains(SrcReg
));
534 unsigned Opc
= AMDGPU::AGPR_32RegClass
.contains(SrcReg
) ?
535 AMDGPU::V_ACCVGPR_READ_B32
: AMDGPU::V_MOV_B32_e32
;
536 BuildMI(MBB
, MI
, DL
, get(Opc
), DestReg
)
537 .addReg(SrcReg
, getKillRegState(KillSrc
));
541 if (RC
== &AMDGPU::SReg_32_XM0RegClass
||
542 RC
== &AMDGPU::SReg_32RegClass
) {
543 if (SrcReg
== AMDGPU::SCC
) {
544 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_CSELECT_B32
), DestReg
)
550 if (DestReg
== AMDGPU::VCC_LO
) {
551 if (AMDGPU::SReg_32RegClass
.contains(SrcReg
)) {
552 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B32
), AMDGPU::VCC_LO
)
553 .addReg(SrcReg
, getKillRegState(KillSrc
));
555 // FIXME: Hack until VReg_1 removed.
556 assert(AMDGPU::VGPR_32RegClass
.contains(SrcReg
));
557 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_CMP_NE_U32_e32
))
559 .addReg(SrcReg
, getKillRegState(KillSrc
));
565 if (!AMDGPU::SReg_32RegClass
.contains(SrcReg
)) {
566 reportIllegalCopy(this, MBB
, MI
, DL
, DestReg
, SrcReg
, KillSrc
);
570 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B32
), DestReg
)
571 .addReg(SrcReg
, getKillRegState(KillSrc
));
575 if (RC
== &AMDGPU::SReg_64RegClass
) {
576 if (DestReg
== AMDGPU::VCC
) {
577 if (AMDGPU::SReg_64RegClass
.contains(SrcReg
)) {
578 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B64
), AMDGPU::VCC
)
579 .addReg(SrcReg
, getKillRegState(KillSrc
));
581 // FIXME: Hack until VReg_1 removed.
582 assert(AMDGPU::VGPR_32RegClass
.contains(SrcReg
));
583 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_CMP_NE_U32_e32
))
585 .addReg(SrcReg
, getKillRegState(KillSrc
));
591 if (!AMDGPU::SReg_64RegClass
.contains(SrcReg
)) {
592 reportIllegalCopy(this, MBB
, MI
, DL
, DestReg
, SrcReg
, KillSrc
);
596 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B64
), DestReg
)
597 .addReg(SrcReg
, getKillRegState(KillSrc
));
601 if (DestReg
== AMDGPU::SCC
) {
602 assert(AMDGPU::SReg_32RegClass
.contains(SrcReg
));
603 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_CMP_LG_U32
))
604 .addReg(SrcReg
, getKillRegState(KillSrc
))
609 if (RC
== &AMDGPU::AGPR_32RegClass
) {
610 assert(AMDGPU::VGPR_32RegClass
.contains(SrcReg
) ||
611 AMDGPU::SReg_32RegClass
.contains(SrcReg
) ||
612 AMDGPU::AGPR_32RegClass
.contains(SrcReg
));
613 if (!AMDGPU::VGPR_32RegClass
.contains(SrcReg
)) {
614 // First try to find defining accvgpr_write to avoid temporary registers.
615 for (auto Def
= MI
, E
= MBB
.begin(); Def
!= E
; ) {
617 if (!Def
->definesRegister(SrcReg
, &RI
))
619 if (Def
->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32
)
622 MachineOperand
&DefOp
= Def
->getOperand(1);
623 assert(DefOp
.isReg() || DefOp
.isImm());
626 // Check that register source operand if not clobbered before MI.
627 // Immediate operands are always safe to propagate.
628 bool SafeToPropagate
= true;
629 for (auto I
= Def
; I
!= MI
&& SafeToPropagate
; ++I
)
630 if (I
->modifiesRegister(DefOp
.getReg(), &RI
))
631 SafeToPropagate
= false;
633 if (!SafeToPropagate
)
636 DefOp
.setIsKill(false);
639 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_ACCVGPR_WRITE_B32
), DestReg
)
645 RS
.enterBasicBlock(MBB
);
648 // Ideally we want to have three registers for a long reg_sequence copy
649 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
650 unsigned MaxVGPRs
= RI
.getRegPressureLimit(&AMDGPU::VGPR_32RegClass
,
653 // Registers in the sequence are allocated contiguously so we can just
654 // use register number to pick one of three round-robin temps.
655 unsigned RegNo
= DestReg
% 3;
656 unsigned Tmp
= RS
.scavengeRegister(&AMDGPU::VGPR_32RegClass
, 0);
658 report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
660 // Only loop through if there are any free registers left, otherwise
661 // scavenger may report a fatal error without emergency spill slot
662 // or spill with the slot.
663 while (RegNo
-- && RS
.FindUnusedReg(&AMDGPU::VGPR_32RegClass
)) {
664 unsigned Tmp2
= RS
.scavengeRegister(&AMDGPU::VGPR_32RegClass
, 0);
665 if (!Tmp2
|| RI
.getHWRegIndex(Tmp2
) >= MaxVGPRs
)
670 copyPhysReg(MBB
, MI
, DL
, Tmp
, SrcReg
, KillSrc
);
671 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_ACCVGPR_WRITE_B32
), DestReg
)
672 .addReg(Tmp
, RegState::Kill
);
676 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_ACCVGPR_WRITE_B32
), DestReg
)
677 .addReg(SrcReg
, getKillRegState(KillSrc
));
681 unsigned EltSize
= 4;
682 unsigned Opcode
= AMDGPU::V_MOV_B32_e32
;
683 if (RI
.isSGPRClass(RC
)) {
684 // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
685 if (!(RI
.getRegSizeInBits(*RC
) % 64)) {
686 Opcode
= AMDGPU::S_MOV_B64
;
689 Opcode
= AMDGPU::S_MOV_B32
;
693 if (!RI
.isSGPRClass(RI
.getPhysRegClass(SrcReg
))) {
694 reportIllegalCopy(this, MBB
, MI
, DL
, DestReg
, SrcReg
, KillSrc
);
697 } else if (RI
.hasAGPRs(RC
)) {
698 Opcode
= RI
.hasVGPRs(RI
.getPhysRegClass(SrcReg
)) ?
699 AMDGPU::V_ACCVGPR_WRITE_B32
: AMDGPU::COPY
;
700 } else if (RI
.hasVGPRs(RC
) && RI
.hasAGPRs(RI
.getPhysRegClass(SrcReg
))) {
701 Opcode
= AMDGPU::V_ACCVGPR_READ_B32
;
704 ArrayRef
<int16_t> SubIndices
= RI
.getRegSplitParts(RC
, EltSize
);
705 bool Forward
= RI
.getHWRegIndex(DestReg
) <= RI
.getHWRegIndex(SrcReg
);
707 for (unsigned Idx
= 0; Idx
< SubIndices
.size(); ++Idx
) {
710 SubIdx
= SubIndices
[Idx
];
712 SubIdx
= SubIndices
[SubIndices
.size() - Idx
- 1];
714 if (Opcode
== TargetOpcode::COPY
) {
715 copyPhysReg(MBB
, MI
, DL
, RI
.getSubReg(DestReg
, SubIdx
),
716 RI
.getSubReg(SrcReg
, SubIdx
), KillSrc
);
720 MachineInstrBuilder Builder
= BuildMI(MBB
, MI
, DL
,
721 get(Opcode
), RI
.getSubReg(DestReg
, SubIdx
));
723 Builder
.addReg(RI
.getSubReg(SrcReg
, SubIdx
));
726 Builder
.addReg(DestReg
, RegState::Define
| RegState::Implicit
);
728 bool UseKill
= KillSrc
&& Idx
== SubIndices
.size() - 1;
729 Builder
.addReg(SrcReg
, getKillRegState(UseKill
) | RegState::Implicit
);
733 int SIInstrInfo::commuteOpcode(unsigned Opcode
) const {
736 // Try to map original to commuted opcode
737 NewOpc
= AMDGPU::getCommuteRev(Opcode
);
739 // Check if the commuted (REV) opcode exists on the target.
740 return pseudoToMCOpcode(NewOpc
) != -1 ? NewOpc
: -1;
742 // Try to map commuted to original opcode
743 NewOpc
= AMDGPU::getCommuteOrig(Opcode
);
745 // Check if the original (non-REV) opcode exists on the target.
746 return pseudoToMCOpcode(NewOpc
) != -1 ? NewOpc
: -1;
751 void SIInstrInfo::materializeImmediate(MachineBasicBlock
&MBB
,
752 MachineBasicBlock::iterator MI
,
753 const DebugLoc
&DL
, unsigned DestReg
,
754 int64_t Value
) const {
755 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
756 const TargetRegisterClass
*RegClass
= MRI
.getRegClass(DestReg
);
757 if (RegClass
== &AMDGPU::SReg_32RegClass
||
758 RegClass
== &AMDGPU::SGPR_32RegClass
||
759 RegClass
== &AMDGPU::SReg_32_XM0RegClass
||
760 RegClass
== &AMDGPU::SReg_32_XM0_XEXECRegClass
) {
761 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B32
), DestReg
)
766 if (RegClass
== &AMDGPU::SReg_64RegClass
||
767 RegClass
== &AMDGPU::SGPR_64RegClass
||
768 RegClass
== &AMDGPU::SReg_64_XEXECRegClass
) {
769 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_MOV_B64
), DestReg
)
774 if (RegClass
== &AMDGPU::VGPR_32RegClass
) {
775 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), DestReg
)
779 if (RegClass
== &AMDGPU::VReg_64RegClass
) {
780 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B64_PSEUDO
), DestReg
)
785 unsigned EltSize
= 4;
786 unsigned Opcode
= AMDGPU::V_MOV_B32_e32
;
787 if (RI
.isSGPRClass(RegClass
)) {
788 if (RI
.getRegSizeInBits(*RegClass
) > 32) {
789 Opcode
= AMDGPU::S_MOV_B64
;
792 Opcode
= AMDGPU::S_MOV_B32
;
797 ArrayRef
<int16_t> SubIndices
= RI
.getRegSplitParts(RegClass
, EltSize
);
798 for (unsigned Idx
= 0; Idx
< SubIndices
.size(); ++Idx
) {
799 int64_t IdxValue
= Idx
== 0 ? Value
: 0;
801 MachineInstrBuilder Builder
= BuildMI(MBB
, MI
, DL
,
802 get(Opcode
), RI
.getSubReg(DestReg
, Idx
));
803 Builder
.addImm(IdxValue
);
807 const TargetRegisterClass
*
808 SIInstrInfo::getPreferredSelectRegClass(unsigned Size
) const {
809 return &AMDGPU::VGPR_32RegClass
;
812 void SIInstrInfo::insertVectorSelect(MachineBasicBlock
&MBB
,
813 MachineBasicBlock::iterator I
,
814 const DebugLoc
&DL
, unsigned DstReg
,
815 ArrayRef
<MachineOperand
> Cond
,
817 unsigned FalseReg
) const {
818 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
819 MachineFunction
*MF
= MBB
.getParent();
820 const GCNSubtarget
&ST
= MF
->getSubtarget
<GCNSubtarget
>();
821 const TargetRegisterClass
*BoolXExecRC
=
822 RI
.getRegClass(AMDGPU::SReg_1_XEXECRegClassID
);
823 assert(MRI
.getRegClass(DstReg
) == &AMDGPU::VGPR_32RegClass
&&
826 if (Cond
.size() == 1) {
827 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
828 BuildMI(MBB
, I
, DL
, get(AMDGPU::COPY
), SReg
)
830 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
836 } else if (Cond
.size() == 2) {
837 assert(Cond
[0].isImm() && "Cond[0] is not an immediate");
838 switch (Cond
[0].getImm()) {
839 case SIInstrInfo::SCC_TRUE
: {
840 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
841 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_CSELECT_B32
842 : AMDGPU::S_CSELECT_B64
), SReg
)
845 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
853 case SIInstrInfo::SCC_FALSE
: {
854 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
855 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_CSELECT_B32
856 : AMDGPU::S_CSELECT_B64
), SReg
)
859 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
867 case SIInstrInfo::VCCNZ
: {
868 MachineOperand RegOp
= Cond
[1];
869 RegOp
.setImplicit(false);
870 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
871 BuildMI(MBB
, I
, DL
, get(AMDGPU::COPY
), SReg
)
873 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
881 case SIInstrInfo::VCCZ
: {
882 MachineOperand RegOp
= Cond
[1];
883 RegOp
.setImplicit(false);
884 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
885 BuildMI(MBB
, I
, DL
, get(AMDGPU::COPY
), SReg
)
887 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
895 case SIInstrInfo::EXECNZ
: {
896 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
897 Register SReg2
= MRI
.createVirtualRegister(RI
.getBoolRC());
898 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
899 : AMDGPU::S_OR_SAVEEXEC_B64
), SReg2
)
901 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_CSELECT_B32
902 : AMDGPU::S_CSELECT_B64
), SReg
)
905 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
913 case SIInstrInfo::EXECZ
: {
914 Register SReg
= MRI
.createVirtualRegister(BoolXExecRC
);
915 Register SReg2
= MRI
.createVirtualRegister(RI
.getBoolRC());
916 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
917 : AMDGPU::S_OR_SAVEEXEC_B64
), SReg2
)
919 BuildMI(MBB
, I
, DL
, get(ST
.isWave32() ? AMDGPU::S_CSELECT_B32
920 : AMDGPU::S_CSELECT_B64
), SReg
)
923 BuildMI(MBB
, I
, DL
, get(AMDGPU::V_CNDMASK_B32_e64
), DstReg
)
929 llvm_unreachable("Unhandled branch predicate EXECZ");
933 llvm_unreachable("invalid branch predicate");
936 llvm_unreachable("Can only handle Cond size 1 or 2");
940 unsigned SIInstrInfo::insertEQ(MachineBasicBlock
*MBB
,
941 MachineBasicBlock::iterator I
,
943 unsigned SrcReg
, int Value
) const {
944 MachineRegisterInfo
&MRI
= MBB
->getParent()->getRegInfo();
945 Register Reg
= MRI
.createVirtualRegister(RI
.getBoolRC());
946 BuildMI(*MBB
, I
, DL
, get(AMDGPU::V_CMP_EQ_I32_e64
), Reg
)
953 unsigned SIInstrInfo::insertNE(MachineBasicBlock
*MBB
,
954 MachineBasicBlock::iterator I
,
956 unsigned SrcReg
, int Value
) const {
957 MachineRegisterInfo
&MRI
= MBB
->getParent()->getRegInfo();
958 Register Reg
= MRI
.createVirtualRegister(RI
.getBoolRC());
959 BuildMI(*MBB
, I
, DL
, get(AMDGPU::V_CMP_NE_I32_e64
), Reg
)
966 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass
*DstRC
) const {
968 if (RI
.hasAGPRs(DstRC
))
970 if (RI
.getRegSizeInBits(*DstRC
) == 32) {
971 return RI
.isSGPRClass(DstRC
) ? AMDGPU::S_MOV_B32
: AMDGPU::V_MOV_B32_e32
;
972 } else if (RI
.getRegSizeInBits(*DstRC
) == 64 && RI
.isSGPRClass(DstRC
)) {
973 return AMDGPU::S_MOV_B64
;
974 } else if (RI
.getRegSizeInBits(*DstRC
) == 64 && !RI
.isSGPRClass(DstRC
)) {
975 return AMDGPU::V_MOV_B64_PSEUDO
;
980 static unsigned getSGPRSpillSaveOpcode(unsigned Size
) {
983 return AMDGPU::SI_SPILL_S32_SAVE
;
985 return AMDGPU::SI_SPILL_S64_SAVE
;
987 return AMDGPU::SI_SPILL_S96_SAVE
;
989 return AMDGPU::SI_SPILL_S128_SAVE
;
991 return AMDGPU::SI_SPILL_S160_SAVE
;
993 return AMDGPU::SI_SPILL_S256_SAVE
;
995 return AMDGPU::SI_SPILL_S512_SAVE
;
997 return AMDGPU::SI_SPILL_S1024_SAVE
;
999 llvm_unreachable("unknown register size");
1003 static unsigned getVGPRSpillSaveOpcode(unsigned Size
) {
1006 return AMDGPU::SI_SPILL_V32_SAVE
;
1008 return AMDGPU::SI_SPILL_V64_SAVE
;
1010 return AMDGPU::SI_SPILL_V96_SAVE
;
1012 return AMDGPU::SI_SPILL_V128_SAVE
;
1014 return AMDGPU::SI_SPILL_V160_SAVE
;
1016 return AMDGPU::SI_SPILL_V256_SAVE
;
1018 return AMDGPU::SI_SPILL_V512_SAVE
;
1020 return AMDGPU::SI_SPILL_V1024_SAVE
;
1022 llvm_unreachable("unknown register size");
1026 static unsigned getAGPRSpillSaveOpcode(unsigned Size
) {
1029 return AMDGPU::SI_SPILL_A32_SAVE
;
1031 return AMDGPU::SI_SPILL_A64_SAVE
;
1033 return AMDGPU::SI_SPILL_A128_SAVE
;
1035 return AMDGPU::SI_SPILL_A512_SAVE
;
1037 return AMDGPU::SI_SPILL_A1024_SAVE
;
1039 llvm_unreachable("unknown register size");
1043 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock
&MBB
,
1044 MachineBasicBlock::iterator MI
,
1045 unsigned SrcReg
, bool isKill
,
1047 const TargetRegisterClass
*RC
,
1048 const TargetRegisterInfo
*TRI
) const {
1049 MachineFunction
*MF
= MBB
.getParent();
1050 SIMachineFunctionInfo
*MFI
= MF
->getInfo
<SIMachineFunctionInfo
>();
1051 MachineFrameInfo
&FrameInfo
= MF
->getFrameInfo();
1052 const DebugLoc
&DL
= MBB
.findDebugLoc(MI
);
1054 unsigned Size
= FrameInfo
.getObjectSize(FrameIndex
);
1055 unsigned Align
= FrameInfo
.getObjectAlignment(FrameIndex
);
1056 MachinePointerInfo PtrInfo
1057 = MachinePointerInfo::getFixedStack(*MF
, FrameIndex
);
1058 MachineMemOperand
*MMO
1059 = MF
->getMachineMemOperand(PtrInfo
, MachineMemOperand::MOStore
,
1061 unsigned SpillSize
= TRI
->getSpillSize(*RC
);
1063 if (RI
.isSGPRClass(RC
)) {
1064 MFI
->setHasSpilledSGPRs();
1066 // We are only allowed to create one new instruction when spilling
1067 // registers, so we need to use pseudo instruction for spilling SGPRs.
1068 const MCInstrDesc
&OpDesc
= get(getSGPRSpillSaveOpcode(SpillSize
));
1070 // The SGPR spill/restore instructions only work on number sgprs, so we need
1071 // to make sure we are using the correct register class.
1072 if (Register::isVirtualRegister(SrcReg
) && SpillSize
== 4) {
1073 MachineRegisterInfo
&MRI
= MF
->getRegInfo();
1074 MRI
.constrainRegClass(SrcReg
, &AMDGPU::SReg_32_XM0RegClass
);
1077 MachineInstrBuilder Spill
= BuildMI(MBB
, MI
, DL
, OpDesc
)
1078 .addReg(SrcReg
, getKillRegState(isKill
)) // data
1079 .addFrameIndex(FrameIndex
) // addr
1081 .addReg(MFI
->getScratchRSrcReg(), RegState::Implicit
)
1082 .addReg(MFI
->getStackPtrOffsetReg(), RegState::Implicit
);
1083 // Add the scratch resource registers as implicit uses because we may end up
1084 // needing them, and need to ensure that the reserved registers are
1085 // correctly handled.
1086 if (RI
.spillSGPRToVGPR())
1087 FrameInfo
.setStackID(FrameIndex
, TargetStackID::SGPRSpill
);
1088 if (ST
.hasScalarStores()) {
1089 // m0 is used for offset to scalar stores if used to spill.
1090 Spill
.addReg(AMDGPU::M0
, RegState::ImplicitDefine
| RegState::Dead
);
1096 unsigned Opcode
= RI
.hasAGPRs(RC
) ? getAGPRSpillSaveOpcode(SpillSize
)
1097 : getVGPRSpillSaveOpcode(SpillSize
);
1098 MFI
->setHasSpilledVGPRs();
1100 auto MIB
= BuildMI(MBB
, MI
, DL
, get(Opcode
));
1101 if (RI
.hasAGPRs(RC
)) {
1102 MachineRegisterInfo
&MRI
= MF
->getRegInfo();
1103 Register Tmp
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
1104 MIB
.addReg(Tmp
, RegState::Define
);
1106 MIB
.addReg(SrcReg
, getKillRegState(isKill
)) // data
1107 .addFrameIndex(FrameIndex
) // addr
1108 .addReg(MFI
->getScratchRSrcReg()) // scratch_rsrc
1109 .addReg(MFI
->getStackPtrOffsetReg()) // scratch_offset
1110 .addImm(0) // offset
1111 .addMemOperand(MMO
);
1114 static unsigned getSGPRSpillRestoreOpcode(unsigned Size
) {
1117 return AMDGPU::SI_SPILL_S32_RESTORE
;
1119 return AMDGPU::SI_SPILL_S64_RESTORE
;
1121 return AMDGPU::SI_SPILL_S96_RESTORE
;
1123 return AMDGPU::SI_SPILL_S128_RESTORE
;
1125 return AMDGPU::SI_SPILL_S160_RESTORE
;
1127 return AMDGPU::SI_SPILL_S256_RESTORE
;
1129 return AMDGPU::SI_SPILL_S512_RESTORE
;
1131 return AMDGPU::SI_SPILL_S1024_RESTORE
;
1133 llvm_unreachable("unknown register size");
1137 static unsigned getVGPRSpillRestoreOpcode(unsigned Size
) {
1140 return AMDGPU::SI_SPILL_V32_RESTORE
;
1142 return AMDGPU::SI_SPILL_V64_RESTORE
;
1144 return AMDGPU::SI_SPILL_V96_RESTORE
;
1146 return AMDGPU::SI_SPILL_V128_RESTORE
;
1148 return AMDGPU::SI_SPILL_V160_RESTORE
;
1150 return AMDGPU::SI_SPILL_V256_RESTORE
;
1152 return AMDGPU::SI_SPILL_V512_RESTORE
;
1154 return AMDGPU::SI_SPILL_V1024_RESTORE
;
1156 llvm_unreachable("unknown register size");
1160 static unsigned getAGPRSpillRestoreOpcode(unsigned Size
) {
1163 return AMDGPU::SI_SPILL_A32_RESTORE
;
1165 return AMDGPU::SI_SPILL_A64_RESTORE
;
1167 return AMDGPU::SI_SPILL_A128_RESTORE
;
1169 return AMDGPU::SI_SPILL_A512_RESTORE
;
1171 return AMDGPU::SI_SPILL_A1024_RESTORE
;
1173 llvm_unreachable("unknown register size");
1177 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock
&MBB
,
1178 MachineBasicBlock::iterator MI
,
1179 unsigned DestReg
, int FrameIndex
,
1180 const TargetRegisterClass
*RC
,
1181 const TargetRegisterInfo
*TRI
) const {
1182 MachineFunction
*MF
= MBB
.getParent();
1183 SIMachineFunctionInfo
*MFI
= MF
->getInfo
<SIMachineFunctionInfo
>();
1184 MachineFrameInfo
&FrameInfo
= MF
->getFrameInfo();
1185 const DebugLoc
&DL
= MBB
.findDebugLoc(MI
);
1186 unsigned Align
= FrameInfo
.getObjectAlignment(FrameIndex
);
1187 unsigned Size
= FrameInfo
.getObjectSize(FrameIndex
);
1188 unsigned SpillSize
= TRI
->getSpillSize(*RC
);
1190 MachinePointerInfo PtrInfo
1191 = MachinePointerInfo::getFixedStack(*MF
, FrameIndex
);
1193 MachineMemOperand
*MMO
= MF
->getMachineMemOperand(
1194 PtrInfo
, MachineMemOperand::MOLoad
, Size
, Align
);
1196 if (RI
.isSGPRClass(RC
)) {
1197 MFI
->setHasSpilledSGPRs();
1199 // FIXME: Maybe this should not include a memoperand because it will be
1200 // lowered to non-memory instructions.
1201 const MCInstrDesc
&OpDesc
= get(getSGPRSpillRestoreOpcode(SpillSize
));
1202 if (Register::isVirtualRegister(DestReg
) && SpillSize
== 4) {
1203 MachineRegisterInfo
&MRI
= MF
->getRegInfo();
1204 MRI
.constrainRegClass(DestReg
, &AMDGPU::SReg_32_XM0RegClass
);
1207 if (RI
.spillSGPRToVGPR())
1208 FrameInfo
.setStackID(FrameIndex
, TargetStackID::SGPRSpill
);
1209 MachineInstrBuilder Spill
= BuildMI(MBB
, MI
, DL
, OpDesc
, DestReg
)
1210 .addFrameIndex(FrameIndex
) // addr
1212 .addReg(MFI
->getScratchRSrcReg(), RegState::Implicit
)
1213 .addReg(MFI
->getStackPtrOffsetReg(), RegState::Implicit
);
1215 if (ST
.hasScalarStores()) {
1216 // m0 is used for offset to scalar stores if used to spill.
1217 Spill
.addReg(AMDGPU::M0
, RegState::ImplicitDefine
| RegState::Dead
);
1223 unsigned Opcode
= RI
.hasAGPRs(RC
) ? getAGPRSpillRestoreOpcode(SpillSize
)
1224 : getVGPRSpillRestoreOpcode(SpillSize
);
1225 auto MIB
= BuildMI(MBB
, MI
, DL
, get(Opcode
), DestReg
);
1226 if (RI
.hasAGPRs(RC
)) {
1227 MachineRegisterInfo
&MRI
= MF
->getRegInfo();
1228 Register Tmp
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
1229 MIB
.addReg(Tmp
, RegState::Define
);
1231 MIB
.addFrameIndex(FrameIndex
) // vaddr
1232 .addReg(MFI
->getScratchRSrcReg()) // scratch_rsrc
1233 .addReg(MFI
->getStackPtrOffsetReg()) // scratch_offset
1234 .addImm(0) // offset
1235 .addMemOperand(MMO
);
1238 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1239 unsigned SIInstrInfo::calculateLDSSpillAddress(
1240 MachineBasicBlock
&MBB
, MachineInstr
&MI
, RegScavenger
*RS
, unsigned TmpReg
,
1241 unsigned FrameOffset
, unsigned Size
) const {
1242 MachineFunction
*MF
= MBB
.getParent();
1243 SIMachineFunctionInfo
*MFI
= MF
->getInfo
<SIMachineFunctionInfo
>();
1244 const GCNSubtarget
&ST
= MF
->getSubtarget
<GCNSubtarget
>();
1245 const DebugLoc
&DL
= MBB
.findDebugLoc(MI
);
1246 unsigned WorkGroupSize
= MFI
->getMaxFlatWorkGroupSize();
1247 unsigned WavefrontSize
= ST
.getWavefrontSize();
1249 unsigned TIDReg
= MFI
->getTIDReg();
1250 if (!MFI
->hasCalculatedTID()) {
1251 MachineBasicBlock
&Entry
= MBB
.getParent()->front();
1252 MachineBasicBlock::iterator Insert
= Entry
.front();
1253 const DebugLoc
&DL
= Insert
->getDebugLoc();
1255 TIDReg
= RI
.findUnusedRegister(MF
->getRegInfo(), &AMDGPU::VGPR_32RegClass
,
1257 if (TIDReg
== AMDGPU::NoRegister
)
1260 if (!AMDGPU::isShader(MF
->getFunction().getCallingConv()) &&
1261 WorkGroupSize
> WavefrontSize
) {
1262 Register TIDIGXReg
=
1263 MFI
->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_X
);
1264 Register TIDIGYReg
=
1265 MFI
->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Y
);
1266 Register TIDIGZReg
=
1267 MFI
->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Z
);
1268 Register InputPtrReg
=
1269 MFI
->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR
);
1270 for (unsigned Reg
: {TIDIGXReg
, TIDIGYReg
, TIDIGZReg
}) {
1271 if (!Entry
.isLiveIn(Reg
))
1272 Entry
.addLiveIn(Reg
);
1275 RS
->enterBasicBlock(Entry
);
1276 // FIXME: Can we scavenge an SReg_64 and access the subregs?
1277 unsigned STmp0
= RS
->scavengeRegister(&AMDGPU::SGPR_32RegClass
, 0);
1278 unsigned STmp1
= RS
->scavengeRegister(&AMDGPU::SGPR_32RegClass
, 0);
1279 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::S_LOAD_DWORD_IMM
), STmp0
)
1280 .addReg(InputPtrReg
)
1281 .addImm(SI::KernelInputOffsets::NGROUPS_Z
);
1282 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::S_LOAD_DWORD_IMM
), STmp1
)
1283 .addReg(InputPtrReg
)
1284 .addImm(SI::KernelInputOffsets::NGROUPS_Y
);
1286 // NGROUPS.X * NGROUPS.Y
1287 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::S_MUL_I32
), STmp1
)
1290 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1291 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::V_MUL_U32_U24_e32
), TIDReg
)
1294 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1295 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::V_MAD_U32_U24
), TIDReg
)
1299 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1300 getAddNoCarry(Entry
, Insert
, DL
, TIDReg
)
1303 .addImm(0); // clamp bit
1306 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::V_MBCNT_LO_U32_B32_e64
),
1311 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::V_MBCNT_HI_U32_B32_e64
),
1317 BuildMI(Entry
, Insert
, DL
, get(AMDGPU::V_LSHLREV_B32_e32
),
1321 MFI
->setTIDReg(TIDReg
);
1324 // Add FrameIndex to LDS offset
1325 unsigned LDSOffset
= MFI
->getLDSSize() + (FrameOffset
* WorkGroupSize
);
1326 getAddNoCarry(MBB
, MI
, DL
, TmpReg
)
1329 .addImm(0); // clamp bit
1334 void SIInstrInfo::insertWaitStates(MachineBasicBlock
&MBB
,
1335 MachineBasicBlock::iterator MI
,
1337 DebugLoc DL
= MBB
.findDebugLoc(MI
);
1345 BuildMI(MBB
, MI
, DL
, get(AMDGPU::S_NOP
))
1350 void SIInstrInfo::insertNoop(MachineBasicBlock
&MBB
,
1351 MachineBasicBlock::iterator MI
) const {
1352 insertWaitStates(MBB
, MI
, 1);
1355 void SIInstrInfo::insertReturn(MachineBasicBlock
&MBB
) const {
1356 auto MF
= MBB
.getParent();
1357 SIMachineFunctionInfo
*Info
= MF
->getInfo
<SIMachineFunctionInfo
>();
1359 assert(Info
->isEntryFunction());
1361 if (MBB
.succ_empty()) {
1362 bool HasNoTerminator
= MBB
.getFirstTerminator() == MBB
.end();
1363 if (HasNoTerminator
) {
1364 if (Info
->returnsVoid()) {
1365 BuildMI(MBB
, MBB
.end(), DebugLoc(), get(AMDGPU::S_ENDPGM
)).addImm(0);
1367 BuildMI(MBB
, MBB
.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG
));
1373 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr
&MI
) {
1374 switch (MI
.getOpcode()) {
1375 default: return 1; // FIXME: Do wait states equal cycles?
1378 return MI
.getOperand(0).getImm() + 1;
1382 bool SIInstrInfo::expandPostRAPseudo(MachineInstr
&MI
) const {
1383 MachineBasicBlock
&MBB
= *MI
.getParent();
1384 DebugLoc DL
= MBB
.findDebugLoc(MI
);
1385 switch (MI
.getOpcode()) {
1386 default: return TargetInstrInfo::expandPostRAPseudo(MI
);
1387 case AMDGPU::S_MOV_B64_term
:
1388 // This is only a terminator to get the correct spill code placement during
1389 // register allocation.
1390 MI
.setDesc(get(AMDGPU::S_MOV_B64
));
1393 case AMDGPU::S_MOV_B32_term
:
1394 // This is only a terminator to get the correct spill code placement during
1395 // register allocation.
1396 MI
.setDesc(get(AMDGPU::S_MOV_B32
));
1399 case AMDGPU::S_XOR_B64_term
:
1400 // This is only a terminator to get the correct spill code placement during
1401 // register allocation.
1402 MI
.setDesc(get(AMDGPU::S_XOR_B64
));
1405 case AMDGPU::S_XOR_B32_term
:
1406 // This is only a terminator to get the correct spill code placement during
1407 // register allocation.
1408 MI
.setDesc(get(AMDGPU::S_XOR_B32
));
1411 case AMDGPU::S_OR_B32_term
:
1412 // This is only a terminator to get the correct spill code placement during
1413 // register allocation.
1414 MI
.setDesc(get(AMDGPU::S_OR_B32
));
1417 case AMDGPU::S_ANDN2_B64_term
:
1418 // This is only a terminator to get the correct spill code placement during
1419 // register allocation.
1420 MI
.setDesc(get(AMDGPU::S_ANDN2_B64
));
1423 case AMDGPU::S_ANDN2_B32_term
:
1424 // This is only a terminator to get the correct spill code placement during
1425 // register allocation.
1426 MI
.setDesc(get(AMDGPU::S_ANDN2_B32
));
1429 case AMDGPU::V_MOV_B64_PSEUDO
: {
1430 Register Dst
= MI
.getOperand(0).getReg();
1431 Register DstLo
= RI
.getSubReg(Dst
, AMDGPU::sub0
);
1432 Register DstHi
= RI
.getSubReg(Dst
, AMDGPU::sub1
);
1434 const MachineOperand
&SrcOp
= MI
.getOperand(1);
1435 // FIXME: Will this work for 64-bit floating point immediates?
1436 assert(!SrcOp
.isFPImm());
1437 if (SrcOp
.isImm()) {
1438 APInt
Imm(64, SrcOp
.getImm());
1439 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), DstLo
)
1440 .addImm(Imm
.getLoBits(32).getZExtValue())
1441 .addReg(Dst
, RegState::Implicit
| RegState::Define
);
1442 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), DstHi
)
1443 .addImm(Imm
.getHiBits(32).getZExtValue())
1444 .addReg(Dst
, RegState::Implicit
| RegState::Define
);
1446 assert(SrcOp
.isReg());
1447 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), DstLo
)
1448 .addReg(RI
.getSubReg(SrcOp
.getReg(), AMDGPU::sub0
))
1449 .addReg(Dst
, RegState::Implicit
| RegState::Define
);
1450 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), DstHi
)
1451 .addReg(RI
.getSubReg(SrcOp
.getReg(), AMDGPU::sub1
))
1452 .addReg(Dst
, RegState::Implicit
| RegState::Define
);
1454 MI
.eraseFromParent();
1457 case AMDGPU::V_SET_INACTIVE_B32
: {
1458 unsigned NotOpc
= ST
.isWave32() ? AMDGPU::S_NOT_B32
: AMDGPU::S_NOT_B64
;
1459 unsigned Exec
= ST
.isWave32() ? AMDGPU::EXEC_LO
: AMDGPU::EXEC
;
1460 BuildMI(MBB
, MI
, DL
, get(NotOpc
), Exec
)
1462 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B32_e32
), MI
.getOperand(0).getReg())
1463 .add(MI
.getOperand(2));
1464 BuildMI(MBB
, MI
, DL
, get(NotOpc
), Exec
)
1466 MI
.eraseFromParent();
1469 case AMDGPU::V_SET_INACTIVE_B64
: {
1470 unsigned NotOpc
= ST
.isWave32() ? AMDGPU::S_NOT_B32
: AMDGPU::S_NOT_B64
;
1471 unsigned Exec
= ST
.isWave32() ? AMDGPU::EXEC_LO
: AMDGPU::EXEC
;
1472 BuildMI(MBB
, MI
, DL
, get(NotOpc
), Exec
)
1474 MachineInstr
*Copy
= BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_MOV_B64_PSEUDO
),
1475 MI
.getOperand(0).getReg())
1476 .add(MI
.getOperand(2));
1477 expandPostRAPseudo(*Copy
);
1478 BuildMI(MBB
, MI
, DL
, get(NotOpc
), Exec
)
1480 MI
.eraseFromParent();
1483 case AMDGPU::V_MOVRELD_B32_V1
:
1484 case AMDGPU::V_MOVRELD_B32_V2
:
1485 case AMDGPU::V_MOVRELD_B32_V4
:
1486 case AMDGPU::V_MOVRELD_B32_V8
:
1487 case AMDGPU::V_MOVRELD_B32_V16
: {
1488 const MCInstrDesc
&MovRelDesc
= get(AMDGPU::V_MOVRELD_B32_e32
);
1489 Register VecReg
= MI
.getOperand(0).getReg();
1490 bool IsUndef
= MI
.getOperand(1).isUndef();
1491 unsigned SubReg
= AMDGPU::sub0
+ MI
.getOperand(3).getImm();
1492 assert(VecReg
== MI
.getOperand(1).getReg());
1494 MachineInstr
*MovRel
=
1495 BuildMI(MBB
, MI
, DL
, MovRelDesc
)
1496 .addReg(RI
.getSubReg(VecReg
, SubReg
), RegState::Undef
)
1497 .add(MI
.getOperand(2))
1498 .addReg(VecReg
, RegState::ImplicitDefine
)
1500 RegState::Implicit
| (IsUndef
? RegState::Undef
: 0));
1502 const int ImpDefIdx
=
1503 MovRelDesc
.getNumOperands() + MovRelDesc
.getNumImplicitUses();
1504 const int ImpUseIdx
= ImpDefIdx
+ 1;
1505 MovRel
->tieOperands(ImpDefIdx
, ImpUseIdx
);
1507 MI
.eraseFromParent();
1510 case AMDGPU::SI_PC_ADD_REL_OFFSET
: {
1511 MachineFunction
&MF
= *MBB
.getParent();
1512 Register Reg
= MI
.getOperand(0).getReg();
1513 Register RegLo
= RI
.getSubReg(Reg
, AMDGPU::sub0
);
1514 Register RegHi
= RI
.getSubReg(Reg
, AMDGPU::sub1
);
1516 // Create a bundle so these instructions won't be re-ordered by the
1517 // post-RA scheduler.
1518 MIBundleBuilder
Bundler(MBB
, MI
);
1519 Bundler
.append(BuildMI(MF
, DL
, get(AMDGPU::S_GETPC_B64
), Reg
));
1521 // Add 32-bit offset from this instruction to the start of the
1523 Bundler
.append(BuildMI(MF
, DL
, get(AMDGPU::S_ADD_U32
), RegLo
)
1525 .add(MI
.getOperand(1)));
1527 MachineInstrBuilder MIB
= BuildMI(MF
, DL
, get(AMDGPU::S_ADDC_U32
), RegHi
)
1529 MIB
.add(MI
.getOperand(2));
1531 Bundler
.append(MIB
);
1532 finalizeBundle(MBB
, Bundler
.begin());
1534 MI
.eraseFromParent();
1537 case AMDGPU::ENTER_WWM
: {
1538 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1540 MI
.setDesc(get(ST
.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1541 : AMDGPU::S_OR_SAVEEXEC_B64
));
1544 case AMDGPU::EXIT_WWM
: {
1545 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1547 MI
.setDesc(get(ST
.isWave32() ? AMDGPU::S_MOV_B32
: AMDGPU::S_MOV_B64
));
1550 case TargetOpcode::BUNDLE
: {
1551 if (!MI
.mayLoad() || MI
.hasUnmodeledSideEffects())
1554 // If it is a load it must be a memory clause
1555 for (MachineBasicBlock::instr_iterator I
= MI
.getIterator();
1556 I
->isBundledWithSucc(); ++I
) {
1557 I
->unbundleFromSucc();
1558 for (MachineOperand
&MO
: I
->operands())
1560 MO
.setIsInternalRead(false);
1563 MI
.eraseFromParent();
1570 bool SIInstrInfo::swapSourceModifiers(MachineInstr
&MI
,
1571 MachineOperand
&Src0
,
1572 unsigned Src0OpName
,
1573 MachineOperand
&Src1
,
1574 unsigned Src1OpName
) const {
1575 MachineOperand
*Src0Mods
= getNamedOperand(MI
, Src0OpName
);
1579 MachineOperand
*Src1Mods
= getNamedOperand(MI
, Src1OpName
);
1581 "All commutable instructions have both src0 and src1 modifiers");
1583 int Src0ModsVal
= Src0Mods
->getImm();
1584 int Src1ModsVal
= Src1Mods
->getImm();
1586 Src1Mods
->setImm(Src0ModsVal
);
1587 Src0Mods
->setImm(Src1ModsVal
);
1591 static MachineInstr
*swapRegAndNonRegOperand(MachineInstr
&MI
,
1592 MachineOperand
&RegOp
,
1593 MachineOperand
&NonRegOp
) {
1594 Register Reg
= RegOp
.getReg();
1595 unsigned SubReg
= RegOp
.getSubReg();
1596 bool IsKill
= RegOp
.isKill();
1597 bool IsDead
= RegOp
.isDead();
1598 bool IsUndef
= RegOp
.isUndef();
1599 bool IsDebug
= RegOp
.isDebug();
1601 if (NonRegOp
.isImm())
1602 RegOp
.ChangeToImmediate(NonRegOp
.getImm());
1603 else if (NonRegOp
.isFI())
1604 RegOp
.ChangeToFrameIndex(NonRegOp
.getIndex());
1608 NonRegOp
.ChangeToRegister(Reg
, false, false, IsKill
, IsDead
, IsUndef
, IsDebug
);
1609 NonRegOp
.setSubReg(SubReg
);
1614 MachineInstr
*SIInstrInfo::commuteInstructionImpl(MachineInstr
&MI
, bool NewMI
,
1616 unsigned Src1Idx
) const {
1617 assert(!NewMI
&& "this should never be used");
1619 unsigned Opc
= MI
.getOpcode();
1620 int CommutedOpcode
= commuteOpcode(Opc
);
1621 if (CommutedOpcode
== -1)
1624 assert(AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src0
) ==
1625 static_cast<int>(Src0Idx
) &&
1626 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src1
) ==
1627 static_cast<int>(Src1Idx
) &&
1628 "inconsistency with findCommutedOpIndices");
1630 MachineOperand
&Src0
= MI
.getOperand(Src0Idx
);
1631 MachineOperand
&Src1
= MI
.getOperand(Src1Idx
);
1633 MachineInstr
*CommutedMI
= nullptr;
1634 if (Src0
.isReg() && Src1
.isReg()) {
1635 if (isOperandLegal(MI
, Src1Idx
, &Src0
)) {
1636 // Be sure to copy the source modifiers to the right place.
1638 = TargetInstrInfo::commuteInstructionImpl(MI
, NewMI
, Src0Idx
, Src1Idx
);
1641 } else if (Src0
.isReg() && !Src1
.isReg()) {
1642 // src0 should always be able to support any operand type, so no need to
1643 // check operand legality.
1644 CommutedMI
= swapRegAndNonRegOperand(MI
, Src0
, Src1
);
1645 } else if (!Src0
.isReg() && Src1
.isReg()) {
1646 if (isOperandLegal(MI
, Src1Idx
, &Src0
))
1647 CommutedMI
= swapRegAndNonRegOperand(MI
, Src1
, Src0
);
1649 // FIXME: Found two non registers to commute. This does happen.
1654 swapSourceModifiers(MI
, Src0
, AMDGPU::OpName::src0_modifiers
,
1655 Src1
, AMDGPU::OpName::src1_modifiers
);
1657 CommutedMI
->setDesc(get(CommutedOpcode
));
1663 // This needs to be implemented because the source modifiers may be inserted
1664 // between the true commutable operands, and the base
1665 // TargetInstrInfo::commuteInstruction uses it.
1666 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr
&MI
,
1667 unsigned &SrcOpIdx0
,
1668 unsigned &SrcOpIdx1
) const {
1669 return findCommutedOpIndices(MI
.getDesc(), SrcOpIdx0
, SrcOpIdx1
);
1672 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc
, unsigned &SrcOpIdx0
,
1673 unsigned &SrcOpIdx1
) const {
1674 if (!Desc
.isCommutable())
1677 unsigned Opc
= Desc
.getOpcode();
1678 int Src0Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src0
);
1682 int Src1Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src1
);
1686 return fixCommutedOpIndices(SrcOpIdx0
, SrcOpIdx1
, Src0Idx
, Src1Idx
);
1689 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp
,
1690 int64_t BrOffset
) const {
1691 // BranchRelaxation should never have to check s_setpc_b64 because its dest
1692 // block is unanalyzable.
1693 assert(BranchOp
!= AMDGPU::S_SETPC_B64
);
1695 // Convert to dwords.
1698 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1699 // from the next instruction.
1702 return isIntN(BranchOffsetBits
, BrOffset
);
1705 MachineBasicBlock
*SIInstrInfo::getBranchDestBlock(
1706 const MachineInstr
&MI
) const {
1707 if (MI
.getOpcode() == AMDGPU::S_SETPC_B64
) {
1708 // This would be a difficult analysis to perform, but can always be legal so
1709 // there's no need to analyze it.
1713 return MI
.getOperand(0).getMBB();
1716 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock
&MBB
,
1717 MachineBasicBlock
&DestBB
,
1720 RegScavenger
*RS
) const {
1721 assert(RS
&& "RegScavenger required for long branching");
1722 assert(MBB
.empty() &&
1723 "new block should be inserted for expanding unconditional branch");
1724 assert(MBB
.pred_size() == 1);
1726 MachineFunction
*MF
= MBB
.getParent();
1727 MachineRegisterInfo
&MRI
= MF
->getRegInfo();
1729 // FIXME: Virtual register workaround for RegScavenger not working with empty
1731 Register PCReg
= MRI
.createVirtualRegister(&AMDGPU::SReg_64RegClass
);
1735 // We need to compute the offset relative to the instruction immediately after
1736 // s_getpc_b64. Insert pc arithmetic code before last terminator.
1737 MachineInstr
*GetPC
= BuildMI(MBB
, I
, DL
, get(AMDGPU::S_GETPC_B64
), PCReg
);
1739 // TODO: Handle > 32-bit block address.
1740 if (BrOffset
>= 0) {
1741 BuildMI(MBB
, I
, DL
, get(AMDGPU::S_ADD_U32
))
1742 .addReg(PCReg
, RegState::Define
, AMDGPU::sub0
)
1743 .addReg(PCReg
, 0, AMDGPU::sub0
)
1744 .addMBB(&DestBB
, MO_LONG_BRANCH_FORWARD
);
1745 BuildMI(MBB
, I
, DL
, get(AMDGPU::S_ADDC_U32
))
1746 .addReg(PCReg
, RegState::Define
, AMDGPU::sub1
)
1747 .addReg(PCReg
, 0, AMDGPU::sub1
)
1750 // Backwards branch.
1751 BuildMI(MBB
, I
, DL
, get(AMDGPU::S_SUB_U32
))
1752 .addReg(PCReg
, RegState::Define
, AMDGPU::sub0
)
1753 .addReg(PCReg
, 0, AMDGPU::sub0
)
1754 .addMBB(&DestBB
, MO_LONG_BRANCH_BACKWARD
);
1755 BuildMI(MBB
, I
, DL
, get(AMDGPU::S_SUBB_U32
))
1756 .addReg(PCReg
, RegState::Define
, AMDGPU::sub1
)
1757 .addReg(PCReg
, 0, AMDGPU::sub1
)
1761 // Insert the indirect branch after the other terminator.
1762 BuildMI(&MBB
, DL
, get(AMDGPU::S_SETPC_B64
))
1765 // FIXME: If spilling is necessary, this will fail because this scavenger has
1766 // no emergency stack slots. It is non-trivial to spill in this situation,
1767 // because the restore code needs to be specially placed after the
1768 // jump. BranchRelaxation then needs to be made aware of the newly inserted
1771 // If a spill is needed for the pc register pair, we need to insert a spill
1772 // restore block right before the destination block, and insert a short branch
1773 // into the old destination block's fallthrough predecessor.
1776 // s_cbranch_scc0 skip_long_branch:
1780 // s_getpc_b64 s[8:9]
1781 // s_add_u32 s8, s8, restore_bb
1782 // s_addc_u32 s9, s9, 0
1783 // s_setpc_b64 s[8:9]
1785 // skip_long_branch:
1790 // dest_bb_fallthrough_predecessor:
1796 // fallthrough dest_bb
1801 RS
->enterBasicBlockEnd(MBB
);
1802 unsigned Scav
= RS
->scavengeRegisterBackwards(
1803 AMDGPU::SReg_64RegClass
,
1804 MachineBasicBlock::iterator(GetPC
), false, 0);
1805 MRI
.replaceRegWith(PCReg
, Scav
);
1806 MRI
.clearVirtRegs();
1807 RS
->setRegUsed(Scav
);
1809 return 4 + 8 + 4 + 4;
1812 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond
) {
1814 case SIInstrInfo::SCC_TRUE
:
1815 return AMDGPU::S_CBRANCH_SCC1
;
1816 case SIInstrInfo::SCC_FALSE
:
1817 return AMDGPU::S_CBRANCH_SCC0
;
1818 case SIInstrInfo::VCCNZ
:
1819 return AMDGPU::S_CBRANCH_VCCNZ
;
1820 case SIInstrInfo::VCCZ
:
1821 return AMDGPU::S_CBRANCH_VCCZ
;
1822 case SIInstrInfo::EXECNZ
:
1823 return AMDGPU::S_CBRANCH_EXECNZ
;
1824 case SIInstrInfo::EXECZ
:
1825 return AMDGPU::S_CBRANCH_EXECZ
;
1827 llvm_unreachable("invalid branch predicate");
1831 SIInstrInfo::BranchPredicate
SIInstrInfo::getBranchPredicate(unsigned Opcode
) {
1833 case AMDGPU::S_CBRANCH_SCC0
:
1835 case AMDGPU::S_CBRANCH_SCC1
:
1837 case AMDGPU::S_CBRANCH_VCCNZ
:
1839 case AMDGPU::S_CBRANCH_VCCZ
:
1841 case AMDGPU::S_CBRANCH_EXECNZ
:
1843 case AMDGPU::S_CBRANCH_EXECZ
:
1850 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock
&MBB
,
1851 MachineBasicBlock::iterator I
,
1852 MachineBasicBlock
*&TBB
,
1853 MachineBasicBlock
*&FBB
,
1854 SmallVectorImpl
<MachineOperand
> &Cond
,
1855 bool AllowModify
) const {
1856 if (I
->getOpcode() == AMDGPU::S_BRANCH
) {
1857 // Unconditional Branch
1858 TBB
= I
->getOperand(0).getMBB();
1862 MachineBasicBlock
*CondBB
= nullptr;
1864 if (I
->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO
) {
1865 CondBB
= I
->getOperand(1).getMBB();
1866 Cond
.push_back(I
->getOperand(0));
1868 BranchPredicate Pred
= getBranchPredicate(I
->getOpcode());
1869 if (Pred
== INVALID_BR
)
1872 CondBB
= I
->getOperand(0).getMBB();
1873 Cond
.push_back(MachineOperand::CreateImm(Pred
));
1874 Cond
.push_back(I
->getOperand(1)); // Save the branch register.
1878 if (I
== MBB
.end()) {
1879 // Conditional branch followed by fall-through.
1884 if (I
->getOpcode() == AMDGPU::S_BRANCH
) {
1886 FBB
= I
->getOperand(0).getMBB();
1893 bool SIInstrInfo::analyzeBranch(MachineBasicBlock
&MBB
, MachineBasicBlock
*&TBB
,
1894 MachineBasicBlock
*&FBB
,
1895 SmallVectorImpl
<MachineOperand
> &Cond
,
1896 bool AllowModify
) const {
1897 MachineBasicBlock::iterator I
= MBB
.getFirstTerminator();
1902 // Skip over the instructions that are artificially terminators for special
1904 while (I
!= E
&& !I
->isBranch() && !I
->isReturn() &&
1905 I
->getOpcode() != AMDGPU::SI_MASK_BRANCH
) {
1906 switch (I
->getOpcode()) {
1907 case AMDGPU::SI_MASK_BRANCH
:
1908 case AMDGPU::S_MOV_B64_term
:
1909 case AMDGPU::S_XOR_B64_term
:
1910 case AMDGPU::S_ANDN2_B64_term
:
1911 case AMDGPU::S_MOV_B32_term
:
1912 case AMDGPU::S_XOR_B32_term
:
1913 case AMDGPU::S_OR_B32_term
:
1914 case AMDGPU::S_ANDN2_B32_term
:
1917 case AMDGPU::SI_ELSE
:
1918 case AMDGPU::SI_KILL_I1_TERMINATOR
:
1919 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR
:
1920 // FIXME: It's messy that these need to be considered here at all.
1923 llvm_unreachable("unexpected non-branch terminator inst");
1932 if (I
->getOpcode() != AMDGPU::SI_MASK_BRANCH
)
1933 return analyzeBranchImpl(MBB
, I
, TBB
, FBB
, Cond
, AllowModify
);
1937 // TODO: Should be able to treat as fallthrough?
1941 if (analyzeBranchImpl(MBB
, I
, TBB
, FBB
, Cond
, AllowModify
))
1944 MachineBasicBlock
*MaskBrDest
= I
->getOperand(0).getMBB();
1946 // Specifically handle the case where the conditional branch is to the same
1947 // destination as the mask branch. e.g.
1949 // si_mask_branch BB8
1950 // s_cbranch_execz BB8
1953 // This is required to understand divergent loops which may need the branches
1955 if (TBB
!= MaskBrDest
|| Cond
.empty())
1958 auto Pred
= Cond
[0].getImm();
1959 return (Pred
!= EXECZ
&& Pred
!= EXECNZ
);
1962 unsigned SIInstrInfo::removeBranch(MachineBasicBlock
&MBB
,
1963 int *BytesRemoved
) const {
1964 MachineBasicBlock::iterator I
= MBB
.getFirstTerminator();
1967 unsigned RemovedSize
= 0;
1968 while (I
!= MBB
.end()) {
1969 MachineBasicBlock::iterator Next
= std::next(I
);
1970 if (I
->getOpcode() == AMDGPU::SI_MASK_BRANCH
) {
1975 RemovedSize
+= getInstSizeInBytes(*I
);
1976 I
->eraseFromParent();
1982 *BytesRemoved
= RemovedSize
;
1987 // Copy the flags onto the implicit condition register operand.
1988 static void preserveCondRegFlags(MachineOperand
&CondReg
,
1989 const MachineOperand
&OrigCond
) {
1990 CondReg
.setIsUndef(OrigCond
.isUndef());
1991 CondReg
.setIsKill(OrigCond
.isKill());
1994 unsigned SIInstrInfo::insertBranch(MachineBasicBlock
&MBB
,
1995 MachineBasicBlock
*TBB
,
1996 MachineBasicBlock
*FBB
,
1997 ArrayRef
<MachineOperand
> Cond
,
1999 int *BytesAdded
) const {
2000 if (!FBB
&& Cond
.empty()) {
2001 BuildMI(&MBB
, DL
, get(AMDGPU::S_BRANCH
))
2008 if(Cond
.size() == 1 && Cond
[0].isReg()) {
2009 BuildMI(&MBB
, DL
, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO
))
2015 assert(TBB
&& Cond
[0].isImm());
2018 = getBranchOpcode(static_cast<BranchPredicate
>(Cond
[0].getImm()));
2022 MachineInstr
*CondBr
=
2023 BuildMI(&MBB
, DL
, get(Opcode
))
2026 // Copy the flags onto the implicit condition register operand.
2027 preserveCondRegFlags(CondBr
->getOperand(1), Cond
[1]);
2036 MachineInstr
*CondBr
=
2037 BuildMI(&MBB
, DL
, get(Opcode
))
2039 BuildMI(&MBB
, DL
, get(AMDGPU::S_BRANCH
))
2042 MachineOperand
&CondReg
= CondBr
->getOperand(1);
2043 CondReg
.setIsUndef(Cond
[1].isUndef());
2044 CondReg
.setIsKill(Cond
[1].isKill());
2052 bool SIInstrInfo::reverseBranchCondition(
2053 SmallVectorImpl
<MachineOperand
> &Cond
) const {
2054 if (Cond
.size() != 2) {
2058 if (Cond
[0].isImm()) {
2059 Cond
[0].setImm(-Cond
[0].getImm());
2066 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock
&MBB
,
2067 ArrayRef
<MachineOperand
> Cond
,
2068 unsigned TrueReg
, unsigned FalseReg
,
2070 int &TrueCycles
, int &FalseCycles
) const {
2071 switch (Cond
[0].getImm()) {
2074 const MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
2075 const TargetRegisterClass
*RC
= MRI
.getRegClass(TrueReg
);
2076 assert(MRI
.getRegClass(FalseReg
) == RC
);
2078 int NumInsts
= AMDGPU::getRegBitWidth(RC
->getID()) / 32;
2079 CondCycles
= TrueCycles
= FalseCycles
= NumInsts
; // ???
2081 // Limit to equal cost for branch vs. N v_cndmask_b32s.
2082 return RI
.hasVGPRs(RC
) && NumInsts
<= 6;
2086 // FIXME: We could insert for VGPRs if we could replace the original compare
2087 // with a vector one.
2088 const MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
2089 const TargetRegisterClass
*RC
= MRI
.getRegClass(TrueReg
);
2090 assert(MRI
.getRegClass(FalseReg
) == RC
);
2092 int NumInsts
= AMDGPU::getRegBitWidth(RC
->getID()) / 32;
2094 // Multiples of 8 can do s_cselect_b64
2095 if (NumInsts
% 2 == 0)
2098 CondCycles
= TrueCycles
= FalseCycles
= NumInsts
; // ???
2099 return RI
.isSGPRClass(RC
);
2106 void SIInstrInfo::insertSelect(MachineBasicBlock
&MBB
,
2107 MachineBasicBlock::iterator I
, const DebugLoc
&DL
,
2108 unsigned DstReg
, ArrayRef
<MachineOperand
> Cond
,
2109 unsigned TrueReg
, unsigned FalseReg
) const {
2110 BranchPredicate Pred
= static_cast<BranchPredicate
>(Cond
[0].getImm());
2111 if (Pred
== VCCZ
|| Pred
== SCC_FALSE
) {
2112 Pred
= static_cast<BranchPredicate
>(-Pred
);
2113 std::swap(TrueReg
, FalseReg
);
2116 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
2117 const TargetRegisterClass
*DstRC
= MRI
.getRegClass(DstReg
);
2118 unsigned DstSize
= RI
.getRegSizeInBits(*DstRC
);
2120 if (DstSize
== 32) {
2121 unsigned SelOp
= Pred
== SCC_TRUE
?
2122 AMDGPU::S_CSELECT_B32
: AMDGPU::V_CNDMASK_B32_e32
;
2124 // Instruction's operands are backwards from what is expected.
2125 MachineInstr
*Select
=
2126 BuildMI(MBB
, I
, DL
, get(SelOp
), DstReg
)
2130 preserveCondRegFlags(Select
->getOperand(3), Cond
[1]);
2134 if (DstSize
== 64 && Pred
== SCC_TRUE
) {
2135 MachineInstr
*Select
=
2136 BuildMI(MBB
, I
, DL
, get(AMDGPU::S_CSELECT_B64
), DstReg
)
2140 preserveCondRegFlags(Select
->getOperand(3), Cond
[1]);
2144 static const int16_t Sub0_15
[] = {
2145 AMDGPU::sub0
, AMDGPU::sub1
, AMDGPU::sub2
, AMDGPU::sub3
,
2146 AMDGPU::sub4
, AMDGPU::sub5
, AMDGPU::sub6
, AMDGPU::sub7
,
2147 AMDGPU::sub8
, AMDGPU::sub9
, AMDGPU::sub10
, AMDGPU::sub11
,
2148 AMDGPU::sub12
, AMDGPU::sub13
, AMDGPU::sub14
, AMDGPU::sub15
,
2151 static const int16_t Sub0_15_64
[] = {
2152 AMDGPU::sub0_sub1
, AMDGPU::sub2_sub3
,
2153 AMDGPU::sub4_sub5
, AMDGPU::sub6_sub7
,
2154 AMDGPU::sub8_sub9
, AMDGPU::sub10_sub11
,
2155 AMDGPU::sub12_sub13
, AMDGPU::sub14_sub15
,
2158 unsigned SelOp
= AMDGPU::V_CNDMASK_B32_e32
;
2159 const TargetRegisterClass
*EltRC
= &AMDGPU::VGPR_32RegClass
;
2160 const int16_t *SubIndices
= Sub0_15
;
2161 int NElts
= DstSize
/ 32;
2163 // 64-bit select is only available for SALU.
2164 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2165 if (Pred
== SCC_TRUE
) {
2167 SelOp
= AMDGPU::S_CSELECT_B32
;
2168 EltRC
= &AMDGPU::SGPR_32RegClass
;
2170 SelOp
= AMDGPU::S_CSELECT_B64
;
2171 EltRC
= &AMDGPU::SGPR_64RegClass
;
2172 SubIndices
= Sub0_15_64
;
2177 MachineInstrBuilder MIB
= BuildMI(
2178 MBB
, I
, DL
, get(AMDGPU::REG_SEQUENCE
), DstReg
);
2180 I
= MIB
->getIterator();
2182 SmallVector
<unsigned, 8> Regs
;
2183 for (int Idx
= 0; Idx
!= NElts
; ++Idx
) {
2184 Register DstElt
= MRI
.createVirtualRegister(EltRC
);
2185 Regs
.push_back(DstElt
);
2187 unsigned SubIdx
= SubIndices
[Idx
];
2189 MachineInstr
*Select
=
2190 BuildMI(MBB
, I
, DL
, get(SelOp
), DstElt
)
2191 .addReg(FalseReg
, 0, SubIdx
)
2192 .addReg(TrueReg
, 0, SubIdx
);
2193 preserveCondRegFlags(Select
->getOperand(3), Cond
[1]);
2194 fixImplicitOperands(*Select
);
2201 bool SIInstrInfo::isFoldableCopy(const MachineInstr
&MI
) const {
2202 switch (MI
.getOpcode()) {
2203 case AMDGPU::V_MOV_B32_e32
:
2204 case AMDGPU::V_MOV_B32_e64
:
2205 case AMDGPU::V_MOV_B64_PSEUDO
: {
2206 // If there are additional implicit register operands, this may be used for
2207 // register indexing so the source register operand isn't simply copied.
2208 unsigned NumOps
= MI
.getDesc().getNumOperands() +
2209 MI
.getDesc().getNumImplicitUses();
2211 return MI
.getNumOperands() == NumOps
;
2213 case AMDGPU::S_MOV_B32
:
2214 case AMDGPU::S_MOV_B64
:
2216 case AMDGPU::V_ACCVGPR_WRITE_B32
:
2217 case AMDGPU::V_ACCVGPR_READ_B32
:
2224 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
2225 unsigned Kind
) const {
2227 case PseudoSourceValue::Stack
:
2228 case PseudoSourceValue::FixedStack
:
2229 return AMDGPUAS::PRIVATE_ADDRESS
;
2230 case PseudoSourceValue::ConstantPool
:
2231 case PseudoSourceValue::GOT
:
2232 case PseudoSourceValue::JumpTable
:
2233 case PseudoSourceValue::GlobalValueCallEntry
:
2234 case PseudoSourceValue::ExternalSymbolCallEntry
:
2235 case PseudoSourceValue::TargetCustom
:
2236 return AMDGPUAS::CONSTANT_ADDRESS
;
2238 return AMDGPUAS::FLAT_ADDRESS
;
2241 static void removeModOperands(MachineInstr
&MI
) {
2242 unsigned Opc
= MI
.getOpcode();
2243 int Src0ModIdx
= AMDGPU::getNamedOperandIdx(Opc
,
2244 AMDGPU::OpName::src0_modifiers
);
2245 int Src1ModIdx
= AMDGPU::getNamedOperandIdx(Opc
,
2246 AMDGPU::OpName::src1_modifiers
);
2247 int Src2ModIdx
= AMDGPU::getNamedOperandIdx(Opc
,
2248 AMDGPU::OpName::src2_modifiers
);
2250 MI
.RemoveOperand(Src2ModIdx
);
2251 MI
.RemoveOperand(Src1ModIdx
);
2252 MI
.RemoveOperand(Src0ModIdx
);
2255 bool SIInstrInfo::FoldImmediate(MachineInstr
&UseMI
, MachineInstr
&DefMI
,
2256 unsigned Reg
, MachineRegisterInfo
*MRI
) const {
2257 if (!MRI
->hasOneNonDBGUse(Reg
))
2260 switch (DefMI
.getOpcode()) {
2263 case AMDGPU::S_MOV_B64
:
2264 // TODO: We could fold 64-bit immediates, but this get compilicated
2265 // when there are sub-registers.
2268 case AMDGPU::V_MOV_B32_e32
:
2269 case AMDGPU::S_MOV_B32
:
2270 case AMDGPU::V_ACCVGPR_WRITE_B32
:
2274 const MachineOperand
*ImmOp
= getNamedOperand(DefMI
, AMDGPU::OpName::src0
);
2276 // FIXME: We could handle FrameIndex values here.
2277 if (!ImmOp
->isImm())
2280 unsigned Opc
= UseMI
.getOpcode();
2281 if (Opc
== AMDGPU::COPY
) {
2282 bool isVGPRCopy
= RI
.isVGPR(*MRI
, UseMI
.getOperand(0).getReg());
2283 unsigned NewOpc
= isVGPRCopy
? AMDGPU::V_MOV_B32_e32
: AMDGPU::S_MOV_B32
;
2284 if (RI
.isAGPR(*MRI
, UseMI
.getOperand(0).getReg())) {
2285 if (!isInlineConstant(*ImmOp
, AMDGPU::OPERAND_REG_INLINE_AC_INT32
))
2287 NewOpc
= AMDGPU::V_ACCVGPR_WRITE_B32
;
2289 UseMI
.setDesc(get(NewOpc
));
2290 UseMI
.getOperand(1).ChangeToImmediate(ImmOp
->getImm());
2291 UseMI
.addImplicitDefUseOperands(*UseMI
.getParent()->getParent());
2295 if (Opc
== AMDGPU::V_MAD_F32
|| Opc
== AMDGPU::V_MAC_F32_e64
||
2296 Opc
== AMDGPU::V_MAD_F16
|| Opc
== AMDGPU::V_MAC_F16_e64
||
2297 Opc
== AMDGPU::V_FMA_F32
|| Opc
== AMDGPU::V_FMAC_F32_e64
||
2298 Opc
== AMDGPU::V_FMA_F16
|| Opc
== AMDGPU::V_FMAC_F16_e64
) {
2299 // Don't fold if we are using source or output modifiers. The new VOP2
2300 // instructions don't have them.
2301 if (hasAnyModifiersSet(UseMI
))
2304 // If this is a free constant, there's no reason to do this.
2305 // TODO: We could fold this here instead of letting SIFoldOperands do it
2307 MachineOperand
*Src0
= getNamedOperand(UseMI
, AMDGPU::OpName::src0
);
2309 // Any src operand can be used for the legality check.
2310 if (isInlineConstant(UseMI
, *Src0
, *ImmOp
))
2313 bool IsF32
= Opc
== AMDGPU::V_MAD_F32
|| Opc
== AMDGPU::V_MAC_F32_e64
||
2314 Opc
== AMDGPU::V_FMA_F32
|| Opc
== AMDGPU::V_FMAC_F32_e64
;
2315 bool IsFMA
= Opc
== AMDGPU::V_FMA_F32
|| Opc
== AMDGPU::V_FMAC_F32_e64
||
2316 Opc
== AMDGPU::V_FMA_F16
|| Opc
== AMDGPU::V_FMAC_F16_e64
;
2317 MachineOperand
*Src1
= getNamedOperand(UseMI
, AMDGPU::OpName::src1
);
2318 MachineOperand
*Src2
= getNamedOperand(UseMI
, AMDGPU::OpName::src2
);
2320 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2321 // We should only expect these to be on src0 due to canonicalizations.
2322 if (Src0
->isReg() && Src0
->getReg() == Reg
) {
2323 if (!Src1
->isReg() || RI
.isSGPRClass(MRI
->getRegClass(Src1
->getReg())))
2326 if (!Src2
->isReg() || RI
.isSGPRClass(MRI
->getRegClass(Src2
->getReg())))
2330 IsFMA
? (IsF32
? AMDGPU::V_FMAMK_F32
: AMDGPU::V_FMAMK_F16
)
2331 : (IsF32
? AMDGPU::V_MADMK_F32
: AMDGPU::V_MADMK_F16
);
2332 if (pseudoToMCOpcode(NewOpc
) == -1)
2335 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2337 const int64_t Imm
= ImmOp
->getImm();
2339 // FIXME: This would be a lot easier if we could return a new instruction
2340 // instead of having to modify in place.
2342 // Remove these first since they are at the end.
2343 UseMI
.RemoveOperand(
2344 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::omod
));
2345 UseMI
.RemoveOperand(
2346 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::clamp
));
2348 Register Src1Reg
= Src1
->getReg();
2349 unsigned Src1SubReg
= Src1
->getSubReg();
2350 Src0
->setReg(Src1Reg
);
2351 Src0
->setSubReg(Src1SubReg
);
2352 Src0
->setIsKill(Src1
->isKill());
2354 if (Opc
== AMDGPU::V_MAC_F32_e64
||
2355 Opc
== AMDGPU::V_MAC_F16_e64
||
2356 Opc
== AMDGPU::V_FMAC_F32_e64
||
2357 Opc
== AMDGPU::V_FMAC_F16_e64
)
2358 UseMI
.untieRegOperand(
2359 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src2
));
2361 Src1
->ChangeToImmediate(Imm
);
2363 removeModOperands(UseMI
);
2364 UseMI
.setDesc(get(NewOpc
));
2366 bool DeleteDef
= MRI
->hasOneNonDBGUse(Reg
);
2368 DefMI
.eraseFromParent();
2373 // Added part is the constant: Use v_madak_{f16, f32}.
2374 if (Src2
->isReg() && Src2
->getReg() == Reg
) {
2375 // Not allowed to use constant bus for another operand.
2376 // We can however allow an inline immediate as src0.
2377 bool Src0Inlined
= false;
2378 if (Src0
->isReg()) {
2379 // Try to inline constant if possible.
2380 // If the Def moves immediate and the use is single
2381 // We are saving VGPR here.
2382 MachineInstr
*Def
= MRI
->getUniqueVRegDef(Src0
->getReg());
2383 if (Def
&& Def
->isMoveImmediate() &&
2384 isInlineConstant(Def
->getOperand(1)) &&
2385 MRI
->hasOneUse(Src0
->getReg())) {
2386 Src0
->ChangeToImmediate(Def
->getOperand(1).getImm());
2388 } else if ((Register::isPhysicalRegister(Src0
->getReg()) &&
2389 (ST
.getConstantBusLimit(Opc
) <= 1 &&
2390 RI
.isSGPRClass(RI
.getPhysRegClass(Src0
->getReg())))) ||
2391 (Register::isVirtualRegister(Src0
->getReg()) &&
2392 (ST
.getConstantBusLimit(Opc
) <= 1 &&
2393 RI
.isSGPRClass(MRI
->getRegClass(Src0
->getReg())))))
2395 // VGPR is okay as Src0 - fallthrough
2398 if (Src1
->isReg() && !Src0Inlined
) {
2399 // We have one slot for inlinable constant so far - try to fill it
2400 MachineInstr
*Def
= MRI
->getUniqueVRegDef(Src1
->getReg());
2401 if (Def
&& Def
->isMoveImmediate() &&
2402 isInlineConstant(Def
->getOperand(1)) &&
2403 MRI
->hasOneUse(Src1
->getReg()) &&
2404 commuteInstruction(UseMI
)) {
2405 Src0
->ChangeToImmediate(Def
->getOperand(1).getImm());
2406 } else if ((Register::isPhysicalRegister(Src1
->getReg()) &&
2407 RI
.isSGPRClass(RI
.getPhysRegClass(Src1
->getReg()))) ||
2408 (Register::isVirtualRegister(Src1
->getReg()) &&
2409 RI
.isSGPRClass(MRI
->getRegClass(Src1
->getReg()))))
2411 // VGPR is okay as Src1 - fallthrough
2415 IsFMA
? (IsF32
? AMDGPU::V_FMAAK_F32
: AMDGPU::V_FMAAK_F16
)
2416 : (IsF32
? AMDGPU::V_MADAK_F32
: AMDGPU::V_MADAK_F16
);
2417 if (pseudoToMCOpcode(NewOpc
) == -1)
2420 const int64_t Imm
= ImmOp
->getImm();
2422 // FIXME: This would be a lot easier if we could return a new instruction
2423 // instead of having to modify in place.
2425 // Remove these first since they are at the end.
2426 UseMI
.RemoveOperand(
2427 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::omod
));
2428 UseMI
.RemoveOperand(
2429 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::clamp
));
2431 if (Opc
== AMDGPU::V_MAC_F32_e64
||
2432 Opc
== AMDGPU::V_MAC_F16_e64
||
2433 Opc
== AMDGPU::V_FMAC_F32_e64
||
2434 Opc
== AMDGPU::V_FMAC_F16_e64
)
2435 UseMI
.untieRegOperand(
2436 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src2
));
2438 // ChangingToImmediate adds Src2 back to the instruction.
2439 Src2
->ChangeToImmediate(Imm
);
2441 // These come before src2.
2442 removeModOperands(UseMI
);
2443 UseMI
.setDesc(get(NewOpc
));
2444 // It might happen that UseMI was commuted
2445 // and we now have SGPR as SRC1. If so 2 inlined
2446 // constant and SGPR are illegal.
2447 legalizeOperands(UseMI
);
2449 bool DeleteDef
= MRI
->hasOneNonDBGUse(Reg
);
2451 DefMI
.eraseFromParent();
2460 static bool offsetsDoNotOverlap(int WidthA
, int OffsetA
,
2461 int WidthB
, int OffsetB
) {
2462 int LowOffset
= OffsetA
< OffsetB
? OffsetA
: OffsetB
;
2463 int HighOffset
= OffsetA
< OffsetB
? OffsetB
: OffsetA
;
2464 int LowWidth
= (LowOffset
== OffsetA
) ? WidthA
: WidthB
;
2465 return LowOffset
+ LowWidth
<= HighOffset
;
2468 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr
&MIa
,
2469 const MachineInstr
&MIb
) const {
2470 const MachineOperand
*BaseOp0
, *BaseOp1
;
2471 int64_t Offset0
, Offset1
;
2473 if (getMemOperandWithOffset(MIa
, BaseOp0
, Offset0
, &RI
) &&
2474 getMemOperandWithOffset(MIb
, BaseOp1
, Offset1
, &RI
)) {
2475 if (!BaseOp0
->isIdenticalTo(*BaseOp1
))
2478 if (!MIa
.hasOneMemOperand() || !MIb
.hasOneMemOperand()) {
2479 // FIXME: Handle ds_read2 / ds_write2.
2482 unsigned Width0
= (*MIa
.memoperands_begin())->getSize();
2483 unsigned Width1
= (*MIb
.memoperands_begin())->getSize();
2484 if (offsetsDoNotOverlap(Width0
, Offset0
, Width1
, Offset1
)) {
2492 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr
&MIa
,
2493 const MachineInstr
&MIb
) const {
2494 assert((MIa
.mayLoad() || MIa
.mayStore()) &&
2495 "MIa must load from or modify a memory location");
2496 assert((MIb
.mayLoad() || MIb
.mayStore()) &&
2497 "MIb must load from or modify a memory location");
2499 if (MIa
.hasUnmodeledSideEffects() || MIb
.hasUnmodeledSideEffects())
2502 // XXX - Can we relax this between address spaces?
2503 if (MIa
.hasOrderedMemoryRef() || MIb
.hasOrderedMemoryRef())
2506 // TODO: Should we check the address space from the MachineMemOperand? That
2507 // would allow us to distinguish objects we know don't alias based on the
2508 // underlying address space, even if it was lowered to a different one,
2509 // e.g. private accesses lowered to use MUBUF instructions on a scratch
2513 return checkInstOffsetsDoNotOverlap(MIa
, MIb
);
2515 return !isFLAT(MIb
) || isSegmentSpecificFLAT(MIb
);
2518 if (isMUBUF(MIa
) || isMTBUF(MIa
)) {
2519 if (isMUBUF(MIb
) || isMTBUF(MIb
))
2520 return checkInstOffsetsDoNotOverlap(MIa
, MIb
);
2522 return !isFLAT(MIb
) && !isSMRD(MIb
);
2527 return checkInstOffsetsDoNotOverlap(MIa
, MIb
);
2529 return !isFLAT(MIb
) && !isMUBUF(MIa
) && !isMTBUF(MIa
);
2534 return checkInstOffsetsDoNotOverlap(MIa
, MIb
);
2542 static int64_t getFoldableImm(const MachineOperand
* MO
) {
2545 const MachineFunction
*MF
= MO
->getParent()->getParent()->getParent();
2546 const MachineRegisterInfo
&MRI
= MF
->getRegInfo();
2547 auto Def
= MRI
.getUniqueVRegDef(MO
->getReg());
2548 if (Def
&& Def
->getOpcode() == AMDGPU::V_MOV_B32_e32
&&
2549 Def
->getOperand(1).isImm())
2550 return Def
->getOperand(1).getImm();
2551 return AMDGPU::NoRegister
;
2554 MachineInstr
*SIInstrInfo::convertToThreeAddress(MachineFunction::iterator
&MBB
,
2556 LiveVariables
*LV
) const {
2557 unsigned Opc
= MI
.getOpcode();
2559 bool IsFMA
= Opc
== AMDGPU::V_FMAC_F32_e32
|| Opc
== AMDGPU::V_FMAC_F32_e64
||
2560 Opc
== AMDGPU::V_FMAC_F16_e32
|| Opc
== AMDGPU::V_FMAC_F16_e64
;
2565 case AMDGPU::V_MAC_F16_e64
:
2566 case AMDGPU::V_FMAC_F16_e64
:
2569 case AMDGPU::V_MAC_F32_e64
:
2570 case AMDGPU::V_FMAC_F32_e64
:
2572 case AMDGPU::V_MAC_F16_e32
:
2573 case AMDGPU::V_FMAC_F16_e32
:
2576 case AMDGPU::V_MAC_F32_e32
:
2577 case AMDGPU::V_FMAC_F32_e32
: {
2578 int Src0Idx
= AMDGPU::getNamedOperandIdx(MI
.getOpcode(),
2579 AMDGPU::OpName::src0
);
2580 const MachineOperand
*Src0
= &MI
.getOperand(Src0Idx
);
2581 if (!Src0
->isReg() && !Src0
->isImm())
2584 if (Src0
->isImm() && !isInlineConstant(MI
, Src0Idx
, *Src0
))
2591 const MachineOperand
*Dst
= getNamedOperand(MI
, AMDGPU::OpName::vdst
);
2592 const MachineOperand
*Src0
= getNamedOperand(MI
, AMDGPU::OpName::src0
);
2593 const MachineOperand
*Src0Mods
=
2594 getNamedOperand(MI
, AMDGPU::OpName::src0_modifiers
);
2595 const MachineOperand
*Src1
= getNamedOperand(MI
, AMDGPU::OpName::src1
);
2596 const MachineOperand
*Src1Mods
=
2597 getNamedOperand(MI
, AMDGPU::OpName::src1_modifiers
);
2598 const MachineOperand
*Src2
= getNamedOperand(MI
, AMDGPU::OpName::src2
);
2599 const MachineOperand
*Clamp
= getNamedOperand(MI
, AMDGPU::OpName::clamp
);
2600 const MachineOperand
*Omod
= getNamedOperand(MI
, AMDGPU::OpName::omod
);
2602 if (!Src0Mods
&& !Src1Mods
&& !Clamp
&& !Omod
&&
2603 // If we have an SGPR input, we will violate the constant bus restriction.
2604 (ST
.getConstantBusLimit(Opc
) > 1 ||
2606 !RI
.isSGPRReg(MBB
->getParent()->getRegInfo(), Src0
->getReg()))) {
2607 if (auto Imm
= getFoldableImm(Src2
)) {
2609 IsFMA
? (IsF16
? AMDGPU::V_FMAAK_F16
: AMDGPU::V_FMAAK_F32
)
2610 : (IsF16
? AMDGPU::V_MADAK_F16
: AMDGPU::V_MADAK_F32
);
2611 if (pseudoToMCOpcode(NewOpc
) != -1)
2612 return BuildMI(*MBB
, MI
, MI
.getDebugLoc(), get(NewOpc
))
2619 IsFMA
? (IsF16
? AMDGPU::V_FMAMK_F16
: AMDGPU::V_FMAMK_F32
)
2620 : (IsF16
? AMDGPU::V_MADMK_F16
: AMDGPU::V_MADMK_F32
);
2621 if (auto Imm
= getFoldableImm(Src1
)) {
2622 if (pseudoToMCOpcode(NewOpc
) != -1)
2623 return BuildMI(*MBB
, MI
, MI
.getDebugLoc(), get(NewOpc
))
2629 if (auto Imm
= getFoldableImm(Src0
)) {
2630 if (pseudoToMCOpcode(NewOpc
) != -1 &&
2631 isOperandLegal(MI
, AMDGPU::getNamedOperandIdx(NewOpc
,
2632 AMDGPU::OpName::src0
), Src1
))
2633 return BuildMI(*MBB
, MI
, MI
.getDebugLoc(), get(NewOpc
))
2641 unsigned NewOpc
= IsFMA
? (IsF16
? AMDGPU::V_FMA_F16
: AMDGPU::V_FMA_F32
)
2642 : (IsF16
? AMDGPU::V_MAD_F16
: AMDGPU::V_MAD_F32
);
2643 if (pseudoToMCOpcode(NewOpc
) == -1)
2646 return BuildMI(*MBB
, MI
, MI
.getDebugLoc(), get(NewOpc
))
2648 .addImm(Src0Mods
? Src0Mods
->getImm() : 0)
2650 .addImm(Src1Mods
? Src1Mods
->getImm() : 0)
2652 .addImm(0) // Src mods
2654 .addImm(Clamp
? Clamp
->getImm() : 0)
2655 .addImm(Omod
? Omod
->getImm() : 0);
2658 // It's not generally safe to move VALU instructions across these since it will
2659 // start using the register as a base index rather than directly.
2660 // XXX - Why isn't hasSideEffects sufficient for these?
2661 static bool changesVGPRIndexingMode(const MachineInstr
&MI
) {
2662 switch (MI
.getOpcode()) {
2663 case AMDGPU::S_SET_GPR_IDX_ON
:
2664 case AMDGPU::S_SET_GPR_IDX_MODE
:
2665 case AMDGPU::S_SET_GPR_IDX_OFF
:
2672 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr
&MI
,
2673 const MachineBasicBlock
*MBB
,
2674 const MachineFunction
&MF
) const {
2675 // XXX - Do we want the SP check in the base implementation?
2677 // Target-independent instructions do not have an implicit-use of EXEC, even
2678 // when they operate on VGPRs. Treating EXEC modifications as scheduling
2679 // boundaries prevents incorrect movements of such instructions.
2680 return TargetInstrInfo::isSchedulingBoundary(MI
, MBB
, MF
) ||
2681 MI
.modifiesRegister(AMDGPU::EXEC
, &RI
) ||
2682 MI
.getOpcode() == AMDGPU::S_SETREG_IMM32_B32
||
2683 MI
.getOpcode() == AMDGPU::S_SETREG_B32
||
2684 MI
.getOpcode() == AMDGPU::S_DENORM_MODE
||
2685 changesVGPRIndexingMode(MI
);
2688 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode
) const {
2689 return Opcode
== AMDGPU::DS_ORDERED_COUNT
||
2690 Opcode
== AMDGPU::DS_GWS_INIT
||
2691 Opcode
== AMDGPU::DS_GWS_SEMA_V
||
2692 Opcode
== AMDGPU::DS_GWS_SEMA_BR
||
2693 Opcode
== AMDGPU::DS_GWS_SEMA_P
||
2694 Opcode
== AMDGPU::DS_GWS_SEMA_RELEASE_ALL
||
2695 Opcode
== AMDGPU::DS_GWS_BARRIER
;
2698 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr
&MI
) const {
2699 unsigned Opcode
= MI
.getOpcode();
2701 if (MI
.mayStore() && isSMRD(MI
))
2702 return true; // scalar store or atomic
2704 // This will terminate the function when other lanes may need to continue.
2708 // These instructions cause shader I/O that may cause hardware lockups
2709 // when executed with an empty EXEC mask.
2711 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2712 // EXEC = 0, but checking for that case here seems not worth it
2713 // given the typical code patterns.
2714 if (Opcode
== AMDGPU::S_SENDMSG
|| Opcode
== AMDGPU::S_SENDMSGHALT
||
2715 Opcode
== AMDGPU::EXP
|| Opcode
== AMDGPU::EXP_DONE
||
2716 Opcode
== AMDGPU::DS_ORDERED_COUNT
|| Opcode
== AMDGPU::S_TRAP
||
2717 Opcode
== AMDGPU::DS_GWS_INIT
|| Opcode
== AMDGPU::DS_GWS_BARRIER
)
2720 if (MI
.isCall() || MI
.isInlineAsm())
2721 return true; // conservative assumption
2723 // These are like SALU instructions in terms of effects, so it's questionable
2724 // whether we should return true for those.
2726 // However, executing them with EXEC = 0 causes them to operate on undefined
2727 // data, which we avoid by returning true here.
2728 if (Opcode
== AMDGPU::V_READFIRSTLANE_B32
|| Opcode
== AMDGPU::V_READLANE_B32
)
2734 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo
&MRI
,
2735 const MachineInstr
&MI
) const {
2736 if (MI
.isMetaInstruction())
2739 // This won't read exec if this is an SGPR->SGPR copy.
2740 if (MI
.isCopyLike()) {
2741 if (!RI
.isSGPRReg(MRI
, MI
.getOperand(0).getReg()))
2744 // Make sure this isn't copying exec as a normal operand
2745 return MI
.readsRegister(AMDGPU::EXEC
, &RI
);
2748 // Make a conservative assumption about the callee.
2752 // Be conservative with any unhandled generic opcodes.
2753 if (!isTargetSpecificOpcode(MI
.getOpcode()))
2756 return !isSALU(MI
) || MI
.readsRegister(AMDGPU::EXEC
, &RI
);
2759 bool SIInstrInfo::isInlineConstant(const APInt
&Imm
) const {
2760 switch (Imm
.getBitWidth()) {
2761 case 1: // This likely will be a condition code mask.
2765 return AMDGPU::isInlinableLiteral32(Imm
.getSExtValue(),
2766 ST
.hasInv2PiInlineImm());
2768 return AMDGPU::isInlinableLiteral64(Imm
.getSExtValue(),
2769 ST
.hasInv2PiInlineImm());
2771 return ST
.has16BitInsts() &&
2772 AMDGPU::isInlinableLiteral16(Imm
.getSExtValue(),
2773 ST
.hasInv2PiInlineImm());
2775 llvm_unreachable("invalid bitwidth");
2779 bool SIInstrInfo::isInlineConstant(const MachineOperand
&MO
,
2780 uint8_t OperandType
) const {
2782 OperandType
< AMDGPU::OPERAND_SRC_FIRST
||
2783 OperandType
> AMDGPU::OPERAND_SRC_LAST
)
2786 // MachineOperand provides no way to tell the true operand size, since it only
2787 // records a 64-bit value. We need to know the size to determine if a 32-bit
2788 // floating point immediate bit pattern is legal for an integer immediate. It
2789 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2791 int64_t Imm
= MO
.getImm();
2792 switch (OperandType
) {
2793 case AMDGPU::OPERAND_REG_IMM_INT32
:
2794 case AMDGPU::OPERAND_REG_IMM_FP32
:
2795 case AMDGPU::OPERAND_REG_INLINE_C_INT32
:
2796 case AMDGPU::OPERAND_REG_INLINE_C_FP32
:
2797 case AMDGPU::OPERAND_REG_INLINE_AC_INT32
:
2798 case AMDGPU::OPERAND_REG_INLINE_AC_FP32
: {
2799 int32_t Trunc
= static_cast<int32_t>(Imm
);
2800 return AMDGPU::isInlinableLiteral32(Trunc
, ST
.hasInv2PiInlineImm());
2802 case AMDGPU::OPERAND_REG_IMM_INT64
:
2803 case AMDGPU::OPERAND_REG_IMM_FP64
:
2804 case AMDGPU::OPERAND_REG_INLINE_C_INT64
:
2805 case AMDGPU::OPERAND_REG_INLINE_C_FP64
:
2806 return AMDGPU::isInlinableLiteral64(MO
.getImm(),
2807 ST
.hasInv2PiInlineImm());
2808 case AMDGPU::OPERAND_REG_IMM_INT16
:
2809 case AMDGPU::OPERAND_REG_IMM_FP16
:
2810 case AMDGPU::OPERAND_REG_INLINE_C_INT16
:
2811 case AMDGPU::OPERAND_REG_INLINE_C_FP16
:
2812 case AMDGPU::OPERAND_REG_INLINE_AC_INT16
:
2813 case AMDGPU::OPERAND_REG_INLINE_AC_FP16
: {
2814 if (isInt
<16>(Imm
) || isUInt
<16>(Imm
)) {
2815 // A few special case instructions have 16-bit operands on subtargets
2816 // where 16-bit instructions are not legal.
2817 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2818 // constants in these cases
2819 int16_t Trunc
= static_cast<int16_t>(Imm
);
2820 return ST
.has16BitInsts() &&
2821 AMDGPU::isInlinableLiteral16(Trunc
, ST
.hasInv2PiInlineImm());
2826 case AMDGPU::OPERAND_REG_IMM_V2INT16
:
2827 case AMDGPU::OPERAND_REG_IMM_V2FP16
:
2828 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16
:
2829 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16
:
2830 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16
:
2831 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16
: {
2832 uint32_t Trunc
= static_cast<uint32_t>(Imm
);
2833 return AMDGPU::isInlinableLiteralV216(Trunc
, ST
.hasInv2PiInlineImm());
2836 llvm_unreachable("invalid bitwidth");
2840 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand
&MO
,
2841 const MCOperandInfo
&OpInfo
) const {
2842 switch (MO
.getType()) {
2843 case MachineOperand::MO_Register
:
2845 case MachineOperand::MO_Immediate
:
2846 return !isInlineConstant(MO
, OpInfo
);
2847 case MachineOperand::MO_FrameIndex
:
2848 case MachineOperand::MO_MachineBasicBlock
:
2849 case MachineOperand::MO_ExternalSymbol
:
2850 case MachineOperand::MO_GlobalAddress
:
2851 case MachineOperand::MO_MCSymbol
:
2854 llvm_unreachable("unexpected operand type");
2858 static bool compareMachineOp(const MachineOperand
&Op0
,
2859 const MachineOperand
&Op1
) {
2860 if (Op0
.getType() != Op1
.getType())
2863 switch (Op0
.getType()) {
2864 case MachineOperand::MO_Register
:
2865 return Op0
.getReg() == Op1
.getReg();
2866 case MachineOperand::MO_Immediate
:
2867 return Op0
.getImm() == Op1
.getImm();
2869 llvm_unreachable("Didn't expect to be comparing these operand types");
2873 bool SIInstrInfo::isImmOperandLegal(const MachineInstr
&MI
, unsigned OpNo
,
2874 const MachineOperand
&MO
) const {
2875 const MCInstrDesc
&InstDesc
= MI
.getDesc();
2876 const MCOperandInfo
&OpInfo
= InstDesc
.OpInfo
[OpNo
];
2878 assert(MO
.isImm() || MO
.isTargetIndex() || MO
.isFI() || MO
.isGlobal());
2880 if (OpInfo
.OperandType
== MCOI::OPERAND_IMMEDIATE
)
2883 if (OpInfo
.RegClass
< 0)
2886 const MachineFunction
*MF
= MI
.getParent()->getParent();
2887 const GCNSubtarget
&ST
= MF
->getSubtarget
<GCNSubtarget
>();
2889 if (MO
.isImm() && isInlineConstant(MO
, OpInfo
)) {
2890 if (isMAI(MI
) && ST
.hasMFMAInlineLiteralBug() &&
2891 OpNo
==(unsigned)AMDGPU::getNamedOperandIdx(MI
.getOpcode(),
2892 AMDGPU::OpName::src2
))
2894 return RI
.opCanUseInlineConstant(OpInfo
.OperandType
);
2897 if (!RI
.opCanUseLiteralConstant(OpInfo
.OperandType
))
2900 if (!isVOP3(MI
) || !AMDGPU::isSISrcOperand(InstDesc
, OpNo
))
2903 return ST
.hasVOP3Literal();
2906 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode
) const {
2907 int Op32
= AMDGPU::getVOPe32(Opcode
);
2911 return pseudoToMCOpcode(Op32
) != -1;
2914 bool SIInstrInfo::hasModifiers(unsigned Opcode
) const {
2915 // The src0_modifier operand is present on all instructions
2916 // that have modifiers.
2918 return AMDGPU::getNamedOperandIdx(Opcode
,
2919 AMDGPU::OpName::src0_modifiers
) != -1;
2922 bool SIInstrInfo::hasModifiersSet(const MachineInstr
&MI
,
2923 unsigned OpName
) const {
2924 const MachineOperand
*Mods
= getNamedOperand(MI
, OpName
);
2925 return Mods
&& Mods
->getImm();
2928 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr
&MI
) const {
2929 return hasModifiersSet(MI
, AMDGPU::OpName::src0_modifiers
) ||
2930 hasModifiersSet(MI
, AMDGPU::OpName::src1_modifiers
) ||
2931 hasModifiersSet(MI
, AMDGPU::OpName::src2_modifiers
) ||
2932 hasModifiersSet(MI
, AMDGPU::OpName::clamp
) ||
2933 hasModifiersSet(MI
, AMDGPU::OpName::omod
);
2936 bool SIInstrInfo::canShrink(const MachineInstr
&MI
,
2937 const MachineRegisterInfo
&MRI
) const {
2938 const MachineOperand
*Src2
= getNamedOperand(MI
, AMDGPU::OpName::src2
);
2939 // Can't shrink instruction with three operands.
2940 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2941 // a special case for it. It can only be shrunk if the third operand
2942 // is vcc, and src0_modifiers and src1_modifiers are not set.
2943 // We should handle this the same way we handle vopc, by addding
2944 // a register allocation hint pre-regalloc and then do the shrinking
2947 switch (MI
.getOpcode()) {
2948 default: return false;
2950 case AMDGPU::V_ADDC_U32_e64
:
2951 case AMDGPU::V_SUBB_U32_e64
:
2952 case AMDGPU::V_SUBBREV_U32_e64
: {
2953 const MachineOperand
*Src1
2954 = getNamedOperand(MI
, AMDGPU::OpName::src1
);
2955 if (!Src1
->isReg() || !RI
.isVGPR(MRI
, Src1
->getReg()))
2957 // Additional verification is needed for sdst/src2.
2960 case AMDGPU::V_MAC_F32_e64
:
2961 case AMDGPU::V_MAC_F16_e64
:
2962 case AMDGPU::V_FMAC_F32_e64
:
2963 case AMDGPU::V_FMAC_F16_e64
:
2964 if (!Src2
->isReg() || !RI
.isVGPR(MRI
, Src2
->getReg()) ||
2965 hasModifiersSet(MI
, AMDGPU::OpName::src2_modifiers
))
2969 case AMDGPU::V_CNDMASK_B32_e64
:
2974 const MachineOperand
*Src1
= getNamedOperand(MI
, AMDGPU::OpName::src1
);
2975 if (Src1
&& (!Src1
->isReg() || !RI
.isVGPR(MRI
, Src1
->getReg()) ||
2976 hasModifiersSet(MI
, AMDGPU::OpName::src1_modifiers
)))
2979 // We don't need to check src0, all input types are legal, so just make sure
2980 // src0 isn't using any modifiers.
2981 if (hasModifiersSet(MI
, AMDGPU::OpName::src0_modifiers
))
2984 // Can it be shrunk to a valid 32 bit opcode?
2985 if (!hasVALU32BitEncoding(MI
.getOpcode()))
2988 // Check output modifiers
2989 return !hasModifiersSet(MI
, AMDGPU::OpName::omod
) &&
2990 !hasModifiersSet(MI
, AMDGPU::OpName::clamp
);
2993 // Set VCC operand with all flags from \p Orig, except for setting it as
2995 static void copyFlagsToImplicitVCC(MachineInstr
&MI
,
2996 const MachineOperand
&Orig
) {
2998 for (MachineOperand
&Use
: MI
.implicit_operands()) {
2999 if (Use
.isUse() && Use
.getReg() == AMDGPU::VCC
) {
3000 Use
.setIsUndef(Orig
.isUndef());
3001 Use
.setIsKill(Orig
.isKill());
3007 MachineInstr
*SIInstrInfo::buildShrunkInst(MachineInstr
&MI
,
3008 unsigned Op32
) const {
3009 MachineBasicBlock
*MBB
= MI
.getParent();;
3010 MachineInstrBuilder Inst32
=
3011 BuildMI(*MBB
, MI
, MI
.getDebugLoc(), get(Op32
));
3013 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3014 // For VOPC instructions, this is replaced by an implicit def of vcc.
3015 int Op32DstIdx
= AMDGPU::getNamedOperandIdx(Op32
, AMDGPU::OpName::vdst
);
3016 if (Op32DstIdx
!= -1) {
3018 Inst32
.add(MI
.getOperand(0));
3020 assert(((MI
.getOperand(0).getReg() == AMDGPU::VCC
) ||
3021 (MI
.getOperand(0).getReg() == AMDGPU::VCC_LO
)) &&
3025 Inst32
.add(*getNamedOperand(MI
, AMDGPU::OpName::src0
));
3027 const MachineOperand
*Src1
= getNamedOperand(MI
, AMDGPU::OpName::src1
);
3031 const MachineOperand
*Src2
= getNamedOperand(MI
, AMDGPU::OpName::src2
);
3034 int Op32Src2Idx
= AMDGPU::getNamedOperandIdx(Op32
, AMDGPU::OpName::src2
);
3035 if (Op32Src2Idx
!= -1) {
3038 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3039 // replaced with an implicit read of vcc. This was already added
3040 // during the initial BuildMI, so find it to preserve the flags.
3041 copyFlagsToImplicitVCC(*Inst32
, *Src2
);
3048 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo
&MRI
,
3049 const MachineOperand
&MO
,
3050 const MCOperandInfo
&OpInfo
) const {
3051 // Literal constants use the constant bus.
3052 //if (isLiteralConstantLike(MO, OpInfo))
3055 return !isInlineConstant(MO
, OpInfo
);
3058 return true; // Misc other operands like FrameIndex
3063 if (Register::isVirtualRegister(MO
.getReg()))
3064 return RI
.isSGPRClass(MRI
.getRegClass(MO
.getReg()));
3067 if (MO
.getReg() == AMDGPU::SGPR_NULL
)
3070 // SGPRs use the constant bus
3071 if (MO
.isImplicit()) {
3072 return MO
.getReg() == AMDGPU::M0
||
3073 MO
.getReg() == AMDGPU::VCC
||
3074 MO
.getReg() == AMDGPU::VCC_LO
;
3076 return AMDGPU::SReg_32RegClass
.contains(MO
.getReg()) ||
3077 AMDGPU::SReg_64RegClass
.contains(MO
.getReg());
3081 static unsigned findImplicitSGPRRead(const MachineInstr
&MI
) {
3082 for (const MachineOperand
&MO
: MI
.implicit_operands()) {
3083 // We only care about reads.
3087 switch (MO
.getReg()) {
3089 case AMDGPU::VCC_LO
:
3090 case AMDGPU::VCC_HI
:
3092 case AMDGPU::FLAT_SCR
:
3100 return AMDGPU::NoRegister
;
3103 static bool shouldReadExec(const MachineInstr
&MI
) {
3104 if (SIInstrInfo::isVALU(MI
)) {
3105 switch (MI
.getOpcode()) {
3106 case AMDGPU::V_READLANE_B32
:
3107 case AMDGPU::V_READLANE_B32_gfx6_gfx7
:
3108 case AMDGPU::V_READLANE_B32_gfx10
:
3109 case AMDGPU::V_READLANE_B32_vi
:
3110 case AMDGPU::V_WRITELANE_B32
:
3111 case AMDGPU::V_WRITELANE_B32_gfx6_gfx7
:
3112 case AMDGPU::V_WRITELANE_B32_gfx10
:
3113 case AMDGPU::V_WRITELANE_B32_vi
:
3120 if (SIInstrInfo::isGenericOpcode(MI
.getOpcode()) ||
3121 SIInstrInfo::isSALU(MI
) ||
3122 SIInstrInfo::isSMRD(MI
))
3128 static bool isSubRegOf(const SIRegisterInfo
&TRI
,
3129 const MachineOperand
&SuperVec
,
3130 const MachineOperand
&SubReg
) {
3131 if (Register::isPhysicalRegister(SubReg
.getReg()))
3132 return TRI
.isSubRegister(SuperVec
.getReg(), SubReg
.getReg());
3134 return SubReg
.getSubReg() != AMDGPU::NoSubRegister
&&
3135 SubReg
.getReg() == SuperVec
.getReg();
3138 bool SIInstrInfo::verifyInstruction(const MachineInstr
&MI
,
3139 StringRef
&ErrInfo
) const {
3140 uint16_t Opcode
= MI
.getOpcode();
3141 if (SIInstrInfo::isGenericOpcode(MI
.getOpcode()))
3144 const MachineFunction
*MF
= MI
.getParent()->getParent();
3145 const MachineRegisterInfo
&MRI
= MF
->getRegInfo();
3147 int Src0Idx
= AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::src0
);
3148 int Src1Idx
= AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::src1
);
3149 int Src2Idx
= AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::src2
);
3151 // Make sure the number of operands is correct.
3152 const MCInstrDesc
&Desc
= get(Opcode
);
3153 if (!Desc
.isVariadic() &&
3154 Desc
.getNumOperands() != MI
.getNumExplicitOperands()) {
3155 ErrInfo
= "Instruction has wrong number of operands.";
3159 if (MI
.isInlineAsm()) {
3160 // Verify register classes for inlineasm constraints.
3161 for (unsigned I
= InlineAsm::MIOp_FirstOperand
, E
= MI
.getNumOperands();
3163 const TargetRegisterClass
*RC
= MI
.getRegClassConstraint(I
, this, &RI
);
3167 const MachineOperand
&Op
= MI
.getOperand(I
);
3171 Register Reg
= Op
.getReg();
3172 if (!Register::isVirtualRegister(Reg
) && !RC
->contains(Reg
)) {
3173 ErrInfo
= "inlineasm operand has incorrect register class.";
3181 // Make sure the register classes are correct.
3182 for (int i
= 0, e
= Desc
.getNumOperands(); i
!= e
; ++i
) {
3183 if (MI
.getOperand(i
).isFPImm()) {
3184 ErrInfo
= "FPImm Machine Operands are not supported. ISel should bitcast "
3185 "all fp values to integers.";
3189 int RegClass
= Desc
.OpInfo
[i
].RegClass
;
3191 switch (Desc
.OpInfo
[i
].OperandType
) {
3192 case MCOI::OPERAND_REGISTER
:
3193 if (MI
.getOperand(i
).isImm() || MI
.getOperand(i
).isGlobal()) {
3194 ErrInfo
= "Illegal immediate value for operand.";
3198 case AMDGPU::OPERAND_REG_IMM_INT32
:
3199 case AMDGPU::OPERAND_REG_IMM_FP32
:
3201 case AMDGPU::OPERAND_REG_INLINE_C_INT32
:
3202 case AMDGPU::OPERAND_REG_INLINE_C_FP32
:
3203 case AMDGPU::OPERAND_REG_INLINE_C_INT64
:
3204 case AMDGPU::OPERAND_REG_INLINE_C_FP64
:
3205 case AMDGPU::OPERAND_REG_INLINE_C_INT16
:
3206 case AMDGPU::OPERAND_REG_INLINE_C_FP16
:
3207 case AMDGPU::OPERAND_REG_INLINE_AC_INT32
:
3208 case AMDGPU::OPERAND_REG_INLINE_AC_FP32
:
3209 case AMDGPU::OPERAND_REG_INLINE_AC_INT16
:
3210 case AMDGPU::OPERAND_REG_INLINE_AC_FP16
: {
3211 const MachineOperand
&MO
= MI
.getOperand(i
);
3212 if (!MO
.isReg() && (!MO
.isImm() || !isInlineConstant(MI
, i
))) {
3213 ErrInfo
= "Illegal immediate value for operand.";
3218 case MCOI::OPERAND_IMMEDIATE
:
3219 case AMDGPU::OPERAND_KIMM32
:
3220 // Check if this operand is an immediate.
3221 // FrameIndex operands will be replaced by immediates, so they are
3223 if (!MI
.getOperand(i
).isImm() && !MI
.getOperand(i
).isFI()) {
3224 ErrInfo
= "Expected immediate, but got non-immediate";
3232 if (!MI
.getOperand(i
).isReg())
3235 if (RegClass
!= -1) {
3236 Register Reg
= MI
.getOperand(i
).getReg();
3237 if (Reg
== AMDGPU::NoRegister
|| Register::isVirtualRegister(Reg
))
3240 const TargetRegisterClass
*RC
= RI
.getRegClass(RegClass
);
3241 if (!RC
->contains(Reg
)) {
3242 ErrInfo
= "Operand has incorrect register class.";
3250 if (!ST
.hasSDWA()) {
3251 ErrInfo
= "SDWA is not supported on this target";
3255 int DstIdx
= AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::vdst
);
3257 const int OpIndicies
[] = { DstIdx
, Src0Idx
, Src1Idx
, Src2Idx
};
3259 for (int OpIdx
: OpIndicies
) {
3262 const MachineOperand
&MO
= MI
.getOperand(OpIdx
);
3264 if (!ST
.hasSDWAScalar()) {
3266 if (!MO
.isReg() || !RI
.hasVGPRs(RI
.getRegClassForReg(MRI
, MO
.getReg()))) {
3267 ErrInfo
= "Only VGPRs allowed as operands in SDWA instructions on VI";
3271 // No immediates on GFX9
3273 ErrInfo
= "Only reg allowed as operands in SDWA instructions on GFX9";
3279 if (!ST
.hasSDWAOmod()) {
3280 // No omod allowed on VI
3281 const MachineOperand
*OMod
= getNamedOperand(MI
, AMDGPU::OpName::omod
);
3282 if (OMod
!= nullptr &&
3283 (!OMod
->isImm() || OMod
->getImm() != 0)) {
3284 ErrInfo
= "OMod not allowed in SDWA instructions on VI";
3289 uint16_t BasicOpcode
= AMDGPU::getBasicFromSDWAOp(Opcode
);
3290 if (isVOPC(BasicOpcode
)) {
3291 if (!ST
.hasSDWASdst() && DstIdx
!= -1) {
3292 // Only vcc allowed as dst on VI for VOPC
3293 const MachineOperand
&Dst
= MI
.getOperand(DstIdx
);
3294 if (!Dst
.isReg() || Dst
.getReg() != AMDGPU::VCC
) {
3295 ErrInfo
= "Only VCC allowed as dst in SDWA instructions on VI";
3298 } else if (!ST
.hasSDWAOutModsVOPC()) {
3299 // No clamp allowed on GFX9 for VOPC
3300 const MachineOperand
*Clamp
= getNamedOperand(MI
, AMDGPU::OpName::clamp
);
3301 if (Clamp
&& (!Clamp
->isImm() || Clamp
->getImm() != 0)) {
3302 ErrInfo
= "Clamp not allowed in VOPC SDWA instructions on VI";
3306 // No omod allowed on GFX9 for VOPC
3307 const MachineOperand
*OMod
= getNamedOperand(MI
, AMDGPU::OpName::omod
);
3308 if (OMod
&& (!OMod
->isImm() || OMod
->getImm() != 0)) {
3309 ErrInfo
= "OMod not allowed in VOPC SDWA instructions on VI";
3315 const MachineOperand
*DstUnused
= getNamedOperand(MI
, AMDGPU::OpName::dst_unused
);
3316 if (DstUnused
&& DstUnused
->isImm() &&
3317 DstUnused
->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE
) {
3318 const MachineOperand
&Dst
= MI
.getOperand(DstIdx
);
3319 if (!Dst
.isReg() || !Dst
.isTied()) {
3320 ErrInfo
= "Dst register should have tied register";
3324 const MachineOperand
&TiedMO
=
3325 MI
.getOperand(MI
.findTiedOperandIdx(DstIdx
));
3326 if (!TiedMO
.isReg() || !TiedMO
.isImplicit() || !TiedMO
.isUse()) {
3328 "Dst register should be tied to implicit use of preserved register";
3330 } else if (Register::isPhysicalRegister(TiedMO
.getReg()) &&
3331 Dst
.getReg() != TiedMO
.getReg()) {
3332 ErrInfo
= "Dst register should use same physical register as preserved";
3339 if (isMIMG(MI
.getOpcode()) && !MI
.mayStore()) {
3340 // Ensure that the return type used is large enough for all the options
3341 // being used TFE/LWE require an extra result register.
3342 const MachineOperand
*DMask
= getNamedOperand(MI
, AMDGPU::OpName::dmask
);
3344 uint64_t DMaskImm
= DMask
->getImm();
3346 isGather4(MI
.getOpcode()) ? 4 : countPopulation(DMaskImm
);
3347 const MachineOperand
*TFE
= getNamedOperand(MI
, AMDGPU::OpName::tfe
);
3348 const MachineOperand
*LWE
= getNamedOperand(MI
, AMDGPU::OpName::lwe
);
3349 const MachineOperand
*D16
= getNamedOperand(MI
, AMDGPU::OpName::d16
);
3351 // Adjust for packed 16 bit values
3352 if (D16
&& D16
->getImm() && !ST
.hasUnpackedD16VMem())
3355 // Adjust if using LWE or TFE
3356 if ((LWE
&& LWE
->getImm()) || (TFE
&& TFE
->getImm()))
3359 const uint32_t DstIdx
=
3360 AMDGPU::getNamedOperandIdx(MI
.getOpcode(), AMDGPU::OpName::vdata
);
3361 const MachineOperand
&Dst
= MI
.getOperand(DstIdx
);
3363 const TargetRegisterClass
*DstRC
= getOpRegClass(MI
, DstIdx
);
3364 uint32_t DstSize
= RI
.getRegSizeInBits(*DstRC
) / 32;
3365 if (RegCount
> DstSize
) {
3366 ErrInfo
= "MIMG instruction returns too many registers for dst "
3374 // Verify VOP*. Ignore multiple sgpr operands on writelane.
3375 if (Desc
.getOpcode() != AMDGPU::V_WRITELANE_B32
3376 && (isVOP1(MI
) || isVOP2(MI
) || isVOP3(MI
) || isVOPC(MI
) || isSDWA(MI
))) {
3377 // Only look at the true operands. Only a real operand can use the constant
3378 // bus, and we don't want to check pseudo-operands like the source modifier
3380 const int OpIndices
[] = { Src0Idx
, Src1Idx
, Src2Idx
};
3382 unsigned ConstantBusCount
= 0;
3383 unsigned LiteralCount
= 0;
3385 if (AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::imm
) != -1)
3388 SmallVector
<unsigned, 2> SGPRsUsed
;
3389 unsigned SGPRUsed
= findImplicitSGPRRead(MI
);
3390 if (SGPRUsed
!= AMDGPU::NoRegister
) {
3392 SGPRsUsed
.push_back(SGPRUsed
);
3395 for (int OpIdx
: OpIndices
) {
3398 const MachineOperand
&MO
= MI
.getOperand(OpIdx
);
3399 if (usesConstantBus(MRI
, MO
, MI
.getDesc().OpInfo
[OpIdx
])) {
3401 SGPRUsed
= MO
.getReg();
3402 if (llvm::all_of(SGPRsUsed
, [this, SGPRUsed
](unsigned SGPR
) {
3403 return !RI
.regsOverlap(SGPRUsed
, SGPR
);
3406 SGPRsUsed
.push_back(SGPRUsed
);
3414 const GCNSubtarget
&ST
= MF
->getSubtarget
<GCNSubtarget
>();
3415 // v_writelane_b32 is an exception from constant bus restriction:
3416 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3417 if (ConstantBusCount
> ST
.getConstantBusLimit(Opcode
) &&
3418 Opcode
!= AMDGPU::V_WRITELANE_B32
) {
3419 ErrInfo
= "VOP* instruction violates constant bus restriction";
3423 if (isVOP3(MI
) && LiteralCount
) {
3424 if (LiteralCount
&& !ST
.hasVOP3Literal()) {
3425 ErrInfo
= "VOP3 instruction uses literal";
3428 if (LiteralCount
> 1) {
3429 ErrInfo
= "VOP3 instruction uses more than one literal";
3435 // Verify misc. restrictions on specific instructions.
3436 if (Desc
.getOpcode() == AMDGPU::V_DIV_SCALE_F32
||
3437 Desc
.getOpcode() == AMDGPU::V_DIV_SCALE_F64
) {
3438 const MachineOperand
&Src0
= MI
.getOperand(Src0Idx
);
3439 const MachineOperand
&Src1
= MI
.getOperand(Src1Idx
);
3440 const MachineOperand
&Src2
= MI
.getOperand(Src2Idx
);
3441 if (Src0
.isReg() && Src1
.isReg() && Src2
.isReg()) {
3442 if (!compareMachineOp(Src0
, Src1
) &&
3443 !compareMachineOp(Src0
, Src2
)) {
3444 ErrInfo
= "v_div_scale_{f32|f64} require src0 = src1 or src2";
3450 if (isSOP2(MI
) || isSOPC(MI
)) {
3451 const MachineOperand
&Src0
= MI
.getOperand(Src0Idx
);
3452 const MachineOperand
&Src1
= MI
.getOperand(Src1Idx
);
3453 unsigned Immediates
= 0;
3455 if (!Src0
.isReg() &&
3456 !isInlineConstant(Src0
, Desc
.OpInfo
[Src0Idx
].OperandType
))
3458 if (!Src1
.isReg() &&
3459 !isInlineConstant(Src1
, Desc
.OpInfo
[Src1Idx
].OperandType
))
3462 if (Immediates
> 1) {
3463 ErrInfo
= "SOP2/SOPC instruction requires too many immediate constants";
3469 auto Op
= getNamedOperand(MI
, AMDGPU::OpName::simm16
);
3470 if (Desc
.isBranch()) {
3472 ErrInfo
= "invalid branch target for SOPK instruction";
3476 uint64_t Imm
= Op
->getImm();
3477 if (sopkIsZext(MI
)) {
3478 if (!isUInt
<16>(Imm
)) {
3479 ErrInfo
= "invalid immediate for SOPK instruction";
3483 if (!isInt
<16>(Imm
)) {
3484 ErrInfo
= "invalid immediate for SOPK instruction";
3491 if (Desc
.getOpcode() == AMDGPU::V_MOVRELS_B32_e32
||
3492 Desc
.getOpcode() == AMDGPU::V_MOVRELS_B32_e64
||
3493 Desc
.getOpcode() == AMDGPU::V_MOVRELD_B32_e32
||
3494 Desc
.getOpcode() == AMDGPU::V_MOVRELD_B32_e64
) {
3495 const bool IsDst
= Desc
.getOpcode() == AMDGPU::V_MOVRELD_B32_e32
||
3496 Desc
.getOpcode() == AMDGPU::V_MOVRELD_B32_e64
;
3498 const unsigned StaticNumOps
= Desc
.getNumOperands() +
3499 Desc
.getNumImplicitUses();
3500 const unsigned NumImplicitOps
= IsDst
? 2 : 1;
3502 // Allow additional implicit operands. This allows a fixup done by the post
3503 // RA scheduler where the main implicit operand is killed and implicit-defs
3504 // are added for sub-registers that remain live after this instruction.
3505 if (MI
.getNumOperands() < StaticNumOps
+ NumImplicitOps
) {
3506 ErrInfo
= "missing implicit register operands";
3510 const MachineOperand
*Dst
= getNamedOperand(MI
, AMDGPU::OpName::vdst
);
3512 if (!Dst
->isUse()) {
3513 ErrInfo
= "v_movreld_b32 vdst should be a use operand";
3518 if (!MI
.isRegTiedToUseOperand(StaticNumOps
, &UseOpIdx
) ||
3519 UseOpIdx
!= StaticNumOps
+ 1) {
3520 ErrInfo
= "movrel implicit operands should be tied";
3525 const MachineOperand
&Src0
= MI
.getOperand(Src0Idx
);
3526 const MachineOperand
&ImpUse
3527 = MI
.getOperand(StaticNumOps
+ NumImplicitOps
- 1);
3528 if (!ImpUse
.isReg() || !ImpUse
.isUse() ||
3529 !isSubRegOf(RI
, ImpUse
, IsDst
? *Dst
: Src0
)) {
3530 ErrInfo
= "src0 should be subreg of implicit vector use";
3535 // Make sure we aren't losing exec uses in the td files. This mostly requires
3536 // being careful when using let Uses to try to add other use registers.
3537 if (shouldReadExec(MI
)) {
3538 if (!MI
.hasRegisterImplicitUseOperand(AMDGPU::EXEC
)) {
3539 ErrInfo
= "VALU instruction does not implicitly read exec mask";
3545 if (MI
.mayStore()) {
3546 // The register offset form of scalar stores may only use m0 as the
3547 // soffset register.
3548 const MachineOperand
*Soff
= getNamedOperand(MI
, AMDGPU::OpName::soff
);
3549 if (Soff
&& Soff
->getReg() != AMDGPU::M0
) {
3550 ErrInfo
= "scalar stores must use m0 as offset register";
3556 if (isFLAT(MI
) && !MF
->getSubtarget
<GCNSubtarget
>().hasFlatInstOffsets()) {
3557 const MachineOperand
*Offset
= getNamedOperand(MI
, AMDGPU::OpName::offset
);
3558 if (Offset
->getImm() != 0) {
3559 ErrInfo
= "subtarget does not support offsets in flat instructions";
3565 const MachineOperand
*DimOp
= getNamedOperand(MI
, AMDGPU::OpName::dim
);
3567 int VAddr0Idx
= AMDGPU::getNamedOperandIdx(Opcode
,
3568 AMDGPU::OpName::vaddr0
);
3569 int SRsrcIdx
= AMDGPU::getNamedOperandIdx(Opcode
, AMDGPU::OpName::srsrc
);
3570 const AMDGPU::MIMGInfo
*Info
= AMDGPU::getMIMGInfo(Opcode
);
3571 const AMDGPU::MIMGBaseOpcodeInfo
*BaseOpcode
=
3572 AMDGPU::getMIMGBaseOpcodeInfo(Info
->BaseOpcode
);
3573 const AMDGPU::MIMGDimInfo
*Dim
=
3574 AMDGPU::getMIMGDimInfoByEncoding(DimOp
->getImm());
3577 ErrInfo
= "dim is out of range";
3581 bool IsNSA
= SRsrcIdx
- VAddr0Idx
> 1;
3582 unsigned AddrWords
= BaseOpcode
->NumExtraArgs
+
3583 (BaseOpcode
->Gradients
? Dim
->NumGradients
: 0) +
3584 (BaseOpcode
->Coordinates
? Dim
->NumCoords
: 0) +
3585 (BaseOpcode
->LodOrClampOrMip
? 1 : 0);
3587 unsigned VAddrWords
;
3589 VAddrWords
= SRsrcIdx
- VAddr0Idx
;
3591 const TargetRegisterClass
*RC
= getOpRegClass(MI
, VAddr0Idx
);
3592 VAddrWords
= MRI
.getTargetRegisterInfo()->getRegSizeInBits(*RC
) / 32;
3595 else if (AddrWords
> 4)
3597 else if (AddrWords
== 3 && VAddrWords
== 4) {
3598 // CodeGen uses the V4 variant of instructions for three addresses,
3599 // because the selection DAG does not support non-power-of-two types.
3604 if (VAddrWords
!= AddrWords
) {
3605 ErrInfo
= "bad vaddr size";
3611 const MachineOperand
*DppCt
= getNamedOperand(MI
, AMDGPU::OpName::dpp_ctrl
);
3613 using namespace AMDGPU::DPP
;
3615 unsigned DC
= DppCt
->getImm();
3616 if (DC
== DppCtrl::DPP_UNUSED1
|| DC
== DppCtrl::DPP_UNUSED2
||
3617 DC
== DppCtrl::DPP_UNUSED3
|| DC
> DppCtrl::DPP_LAST
||
3618 (DC
>= DppCtrl::DPP_UNUSED4_FIRST
&& DC
<= DppCtrl::DPP_UNUSED4_LAST
) ||
3619 (DC
>= DppCtrl::DPP_UNUSED5_FIRST
&& DC
<= DppCtrl::DPP_UNUSED5_LAST
) ||
3620 (DC
>= DppCtrl::DPP_UNUSED6_FIRST
&& DC
<= DppCtrl::DPP_UNUSED6_LAST
) ||
3621 (DC
>= DppCtrl::DPP_UNUSED7_FIRST
&& DC
<= DppCtrl::DPP_UNUSED7_LAST
) ||
3622 (DC
>= DppCtrl::DPP_UNUSED8_FIRST
&& DC
<= DppCtrl::DPP_UNUSED8_LAST
)) {
3623 ErrInfo
= "Invalid dpp_ctrl value";
3626 if (DC
>= DppCtrl::WAVE_SHL1
&& DC
<= DppCtrl::WAVE_ROR1
&&
3627 ST
.getGeneration() >= AMDGPUSubtarget::GFX10
) {
3628 ErrInfo
= "Invalid dpp_ctrl value: "
3629 "wavefront shifts are not supported on GFX10+";
3632 if (DC
>= DppCtrl::BCAST15
&& DC
<= DppCtrl::BCAST31
&&
3633 ST
.getGeneration() >= AMDGPUSubtarget::GFX10
) {
3634 ErrInfo
= "Invalid dpp_ctrl value: "
3635 "broadcasts are not supported on GFX10+";
3638 if (DC
>= DppCtrl::ROW_SHARE_FIRST
&& DC
<= DppCtrl::ROW_XMASK_LAST
&&
3639 ST
.getGeneration() < AMDGPUSubtarget::GFX10
) {
3640 ErrInfo
= "Invalid dpp_ctrl value: "
3641 "row_share and row_xmask are not supported before GFX10";
3649 unsigned SIInstrInfo::getVALUOp(const MachineInstr
&MI
) const {
3650 switch (MI
.getOpcode()) {
3651 default: return AMDGPU::INSTRUCTION_LIST_END
;
3652 case AMDGPU::REG_SEQUENCE
: return AMDGPU::REG_SEQUENCE
;
3653 case AMDGPU::COPY
: return AMDGPU::COPY
;
3654 case AMDGPU::PHI
: return AMDGPU::PHI
;
3655 case AMDGPU::INSERT_SUBREG
: return AMDGPU::INSERT_SUBREG
;
3656 case AMDGPU::WQM
: return AMDGPU::WQM
;
3657 case AMDGPU::SOFT_WQM
: return AMDGPU::SOFT_WQM
;
3658 case AMDGPU::WWM
: return AMDGPU::WWM
;
3659 case AMDGPU::S_MOV_B32
: {
3660 const MachineRegisterInfo
&MRI
= MI
.getParent()->getParent()->getRegInfo();
3661 return MI
.getOperand(1).isReg() ||
3662 RI
.isAGPR(MRI
, MI
.getOperand(0).getReg()) ?
3663 AMDGPU::COPY
: AMDGPU::V_MOV_B32_e32
;
3665 case AMDGPU::S_ADD_I32
:
3666 return ST
.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64
: AMDGPU::V_ADD_I32_e32
;
3667 case AMDGPU::S_ADDC_U32
:
3668 return AMDGPU::V_ADDC_U32_e32
;
3669 case AMDGPU::S_SUB_I32
:
3670 return ST
.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64
: AMDGPU::V_SUB_I32_e32
;
3671 // FIXME: These are not consistently handled, and selected when the carry is
3673 case AMDGPU::S_ADD_U32
:
3674 return AMDGPU::V_ADD_I32_e32
;
3675 case AMDGPU::S_SUB_U32
:
3676 return AMDGPU::V_SUB_I32_e32
;
3677 case AMDGPU::S_SUBB_U32
: return AMDGPU::V_SUBB_U32_e32
;
3678 case AMDGPU::S_MUL_I32
: return AMDGPU::V_MUL_LO_U32
;
3679 case AMDGPU::S_MUL_HI_U32
: return AMDGPU::V_MUL_HI_U32
;
3680 case AMDGPU::S_MUL_HI_I32
: return AMDGPU::V_MUL_HI_I32
;
3681 case AMDGPU::S_AND_B32
: return AMDGPU::V_AND_B32_e64
;
3682 case AMDGPU::S_OR_B32
: return AMDGPU::V_OR_B32_e64
;
3683 case AMDGPU::S_XOR_B32
: return AMDGPU::V_XOR_B32_e64
;
3684 case AMDGPU::S_XNOR_B32
:
3685 return ST
.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64
: AMDGPU::INSTRUCTION_LIST_END
;
3686 case AMDGPU::S_MIN_I32
: return AMDGPU::V_MIN_I32_e64
;
3687 case AMDGPU::S_MIN_U32
: return AMDGPU::V_MIN_U32_e64
;
3688 case AMDGPU::S_MAX_I32
: return AMDGPU::V_MAX_I32_e64
;
3689 case AMDGPU::S_MAX_U32
: return AMDGPU::V_MAX_U32_e64
;
3690 case AMDGPU::S_ASHR_I32
: return AMDGPU::V_ASHR_I32_e32
;
3691 case AMDGPU::S_ASHR_I64
: return AMDGPU::V_ASHR_I64
;
3692 case AMDGPU::S_LSHL_B32
: return AMDGPU::V_LSHL_B32_e32
;
3693 case AMDGPU::S_LSHL_B64
: return AMDGPU::V_LSHL_B64
;
3694 case AMDGPU::S_LSHR_B32
: return AMDGPU::V_LSHR_B32_e32
;
3695 case AMDGPU::S_LSHR_B64
: return AMDGPU::V_LSHR_B64
;
3696 case AMDGPU::S_SEXT_I32_I8
: return AMDGPU::V_BFE_I32
;
3697 case AMDGPU::S_SEXT_I32_I16
: return AMDGPU::V_BFE_I32
;
3698 case AMDGPU::S_BFE_U32
: return AMDGPU::V_BFE_U32
;
3699 case AMDGPU::S_BFE_I32
: return AMDGPU::V_BFE_I32
;
3700 case AMDGPU::S_BFM_B32
: return AMDGPU::V_BFM_B32_e64
;
3701 case AMDGPU::S_BREV_B32
: return AMDGPU::V_BFREV_B32_e32
;
3702 case AMDGPU::S_NOT_B32
: return AMDGPU::V_NOT_B32_e32
;
3703 case AMDGPU::S_NOT_B64
: return AMDGPU::V_NOT_B32_e32
;
3704 case AMDGPU::S_CMP_EQ_I32
: return AMDGPU::V_CMP_EQ_I32_e32
;
3705 case AMDGPU::S_CMP_LG_I32
: return AMDGPU::V_CMP_NE_I32_e32
;
3706 case AMDGPU::S_CMP_GT_I32
: return AMDGPU::V_CMP_GT_I32_e32
;
3707 case AMDGPU::S_CMP_GE_I32
: return AMDGPU::V_CMP_GE_I32_e32
;
3708 case AMDGPU::S_CMP_LT_I32
: return AMDGPU::V_CMP_LT_I32_e32
;
3709 case AMDGPU::S_CMP_LE_I32
: return AMDGPU::V_CMP_LE_I32_e32
;
3710 case AMDGPU::S_CMP_EQ_U32
: return AMDGPU::V_CMP_EQ_U32_e32
;
3711 case AMDGPU::S_CMP_LG_U32
: return AMDGPU::V_CMP_NE_U32_e32
;
3712 case AMDGPU::S_CMP_GT_U32
: return AMDGPU::V_CMP_GT_U32_e32
;
3713 case AMDGPU::S_CMP_GE_U32
: return AMDGPU::V_CMP_GE_U32_e32
;
3714 case AMDGPU::S_CMP_LT_U32
: return AMDGPU::V_CMP_LT_U32_e32
;
3715 case AMDGPU::S_CMP_LE_U32
: return AMDGPU::V_CMP_LE_U32_e32
;
3716 case AMDGPU::S_CMP_EQ_U64
: return AMDGPU::V_CMP_EQ_U64_e32
;
3717 case AMDGPU::S_CMP_LG_U64
: return AMDGPU::V_CMP_NE_U64_e32
;
3718 case AMDGPU::S_BCNT1_I32_B32
: return AMDGPU::V_BCNT_U32_B32_e64
;
3719 case AMDGPU::S_FF1_I32_B32
: return AMDGPU::V_FFBL_B32_e32
;
3720 case AMDGPU::S_FLBIT_I32_B32
: return AMDGPU::V_FFBH_U32_e32
;
3721 case AMDGPU::S_FLBIT_I32
: return AMDGPU::V_FFBH_I32_e64
;
3722 case AMDGPU::S_CBRANCH_SCC0
: return AMDGPU::S_CBRANCH_VCCZ
;
3723 case AMDGPU::S_CBRANCH_SCC1
: return AMDGPU::S_CBRANCH_VCCNZ
;
3726 "Unexpected scalar opcode without corresponding vector one!");
3729 const TargetRegisterClass
*SIInstrInfo::getOpRegClass(const MachineInstr
&MI
,
3730 unsigned OpNo
) const {
3731 const MachineRegisterInfo
&MRI
= MI
.getParent()->getParent()->getRegInfo();
3732 const MCInstrDesc
&Desc
= get(MI
.getOpcode());
3733 if (MI
.isVariadic() || OpNo
>= Desc
.getNumOperands() ||
3734 Desc
.OpInfo
[OpNo
].RegClass
== -1) {
3735 Register Reg
= MI
.getOperand(OpNo
).getReg();
3737 if (Register::isVirtualRegister(Reg
))
3738 return MRI
.getRegClass(Reg
);
3739 return RI
.getPhysRegClass(Reg
);
3742 unsigned RCID
= Desc
.OpInfo
[OpNo
].RegClass
;
3743 return RI
.getRegClass(RCID
);
3746 void SIInstrInfo::legalizeOpWithMove(MachineInstr
&MI
, unsigned OpIdx
) const {
3747 MachineBasicBlock::iterator I
= MI
;
3748 MachineBasicBlock
*MBB
= MI
.getParent();
3749 MachineOperand
&MO
= MI
.getOperand(OpIdx
);
3750 MachineRegisterInfo
&MRI
= MBB
->getParent()->getRegInfo();
3751 const SIRegisterInfo
*TRI
=
3752 static_cast<const SIRegisterInfo
*>(MRI
.getTargetRegisterInfo());
3753 unsigned RCID
= get(MI
.getOpcode()).OpInfo
[OpIdx
].RegClass
;
3754 const TargetRegisterClass
*RC
= RI
.getRegClass(RCID
);
3755 unsigned Size
= TRI
->getRegSizeInBits(*RC
);
3756 unsigned Opcode
= (Size
== 64) ? AMDGPU::V_MOV_B64_PSEUDO
: AMDGPU::V_MOV_B32_e32
;
3758 Opcode
= AMDGPU::COPY
;
3759 else if (RI
.isSGPRClass(RC
))
3760 Opcode
= (Size
== 64) ? AMDGPU::S_MOV_B64
: AMDGPU::S_MOV_B32
;
3762 const TargetRegisterClass
*VRC
= RI
.getEquivalentVGPRClass(RC
);
3763 if (RI
.getCommonSubClass(&AMDGPU::VReg_64RegClass
, VRC
))
3764 VRC
= &AMDGPU::VReg_64RegClass
;
3766 VRC
= &AMDGPU::VGPR_32RegClass
;
3768 Register Reg
= MRI
.createVirtualRegister(VRC
);
3769 DebugLoc DL
= MBB
->findDebugLoc(I
);
3770 BuildMI(*MI
.getParent(), I
, DL
, get(Opcode
), Reg
).add(MO
);
3771 MO
.ChangeToRegister(Reg
, false);
3774 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI
,
3775 MachineRegisterInfo
&MRI
,
3776 MachineOperand
&SuperReg
,
3777 const TargetRegisterClass
*SuperRC
,
3779 const TargetRegisterClass
*SubRC
)
3781 MachineBasicBlock
*MBB
= MI
->getParent();
3782 DebugLoc DL
= MI
->getDebugLoc();
3783 Register SubReg
= MRI
.createVirtualRegister(SubRC
);
3785 if (SuperReg
.getSubReg() == AMDGPU::NoSubRegister
) {
3786 BuildMI(*MBB
, MI
, DL
, get(TargetOpcode::COPY
), SubReg
)
3787 .addReg(SuperReg
.getReg(), 0, SubIdx
);
3791 // Just in case the super register is itself a sub-register, copy it to a new
3792 // value so we don't need to worry about merging its subreg index with the
3793 // SubIdx passed to this function. The register coalescer should be able to
3794 // eliminate this extra copy.
3795 Register NewSuperReg
= MRI
.createVirtualRegister(SuperRC
);
3797 BuildMI(*MBB
, MI
, DL
, get(TargetOpcode::COPY
), NewSuperReg
)
3798 .addReg(SuperReg
.getReg(), 0, SuperReg
.getSubReg());
3800 BuildMI(*MBB
, MI
, DL
, get(TargetOpcode::COPY
), SubReg
)
3801 .addReg(NewSuperReg
, 0, SubIdx
);
3806 MachineOperand
SIInstrInfo::buildExtractSubRegOrImm(
3807 MachineBasicBlock::iterator MII
,
3808 MachineRegisterInfo
&MRI
,
3810 const TargetRegisterClass
*SuperRC
,
3812 const TargetRegisterClass
*SubRC
) const {
3814 if (SubIdx
== AMDGPU::sub0
)
3815 return MachineOperand::CreateImm(static_cast<int32_t>(Op
.getImm()));
3816 if (SubIdx
== AMDGPU::sub1
)
3817 return MachineOperand::CreateImm(static_cast<int32_t>(Op
.getImm() >> 32));
3819 llvm_unreachable("Unhandled register index for immediate");
3822 unsigned SubReg
= buildExtractSubReg(MII
, MRI
, Op
, SuperRC
,
3824 return MachineOperand::CreateReg(SubReg
, false);
3827 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3828 void SIInstrInfo::swapOperands(MachineInstr
&Inst
) const {
3829 assert(Inst
.getNumExplicitOperands() == 3);
3830 MachineOperand Op1
= Inst
.getOperand(1);
3831 Inst
.RemoveOperand(1);
3832 Inst
.addOperand(Op1
);
3835 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo
&MRI
,
3836 const MCOperandInfo
&OpInfo
,
3837 const MachineOperand
&MO
) const {
3841 Register Reg
= MO
.getReg();
3842 const TargetRegisterClass
*RC
= Register::isVirtualRegister(Reg
)
3843 ? MRI
.getRegClass(Reg
)
3844 : RI
.getPhysRegClass(Reg
);
3846 const SIRegisterInfo
*TRI
=
3847 static_cast<const SIRegisterInfo
*>(MRI
.getTargetRegisterInfo());
3848 RC
= TRI
->getSubRegClass(RC
, MO
.getSubReg());
3850 // In order to be legal, the common sub-class must be equal to the
3851 // class of the current operand. For example:
3853 // v_mov_b32 s0 ; Operand defined as vsrc_b32
3854 // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3856 // s_sendmsg 0, s0 ; Operand defined as m0reg
3857 // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3859 return RI
.getCommonSubClass(RC
, RI
.getRegClass(OpInfo
.RegClass
)) == RC
;
3862 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo
&MRI
,
3863 const MCOperandInfo
&OpInfo
,
3864 const MachineOperand
&MO
) const {
3866 return isLegalRegOperand(MRI
, OpInfo
, MO
);
3868 // Handle non-register types that are treated like immediates.
3869 assert(MO
.isImm() || MO
.isTargetIndex() || MO
.isFI() || MO
.isGlobal());
3873 bool SIInstrInfo::isOperandLegal(const MachineInstr
&MI
, unsigned OpIdx
,
3874 const MachineOperand
*MO
) const {
3875 const MachineFunction
&MF
= *MI
.getParent()->getParent();
3876 const MachineRegisterInfo
&MRI
= MF
.getRegInfo();
3877 const MCInstrDesc
&InstDesc
= MI
.getDesc();
3878 const MCOperandInfo
&OpInfo
= InstDesc
.OpInfo
[OpIdx
];
3879 const GCNSubtarget
&ST
= MF
.getSubtarget
<GCNSubtarget
>();
3880 const TargetRegisterClass
*DefinedRC
=
3881 OpInfo
.RegClass
!= -1 ? RI
.getRegClass(OpInfo
.RegClass
) : nullptr;
3883 MO
= &MI
.getOperand(OpIdx
);
3885 int ConstantBusLimit
= ST
.getConstantBusLimit(MI
.getOpcode());
3886 int VOP3LiteralLimit
= ST
.hasVOP3Literal() ? 1 : 0;
3887 if (isVALU(MI
) && usesConstantBus(MRI
, *MO
, OpInfo
)) {
3888 if (isVOP3(MI
) && isLiteralConstantLike(*MO
, OpInfo
) && !VOP3LiteralLimit
--)
3891 SmallDenseSet
<RegSubRegPair
> SGPRsUsed
;
3893 SGPRsUsed
.insert(RegSubRegPair(MO
->getReg(), MO
->getSubReg()));
3895 for (unsigned i
= 0, e
= MI
.getNumOperands(); i
!= e
; ++i
) {
3898 const MachineOperand
&Op
= MI
.getOperand(i
);
3900 RegSubRegPair
SGPR(Op
.getReg(), Op
.getSubReg());
3901 if (!SGPRsUsed
.count(SGPR
) &&
3902 usesConstantBus(MRI
, Op
, InstDesc
.OpInfo
[i
])) {
3903 if (--ConstantBusLimit
<= 0)
3905 SGPRsUsed
.insert(SGPR
);
3907 } else if (InstDesc
.OpInfo
[i
].OperandType
== AMDGPU::OPERAND_KIMM32
) {
3908 if (--ConstantBusLimit
<= 0)
3910 } else if (isVOP3(MI
) && AMDGPU::isSISrcOperand(InstDesc
, i
) &&
3911 isLiteralConstantLike(Op
, InstDesc
.OpInfo
[i
])) {
3912 if (!VOP3LiteralLimit
--)
3914 if (--ConstantBusLimit
<= 0)
3922 return isLegalRegOperand(MRI
, OpInfo
, *MO
);
3925 // Handle non-register types that are treated like immediates.
3926 assert(MO
->isImm() || MO
->isTargetIndex() || MO
->isFI() || MO
->isGlobal());
3929 // This operand expects an immediate.
3933 return isImmOperandLegal(MI
, OpIdx
, *MO
);
3936 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo
&MRI
,
3937 MachineInstr
&MI
) const {
3938 unsigned Opc
= MI
.getOpcode();
3939 const MCInstrDesc
&InstrDesc
= get(Opc
);
3941 int Src0Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src0
);
3942 MachineOperand
&Src0
= MI
.getOperand(Src0Idx
);
3944 int Src1Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src1
);
3945 MachineOperand
&Src1
= MI
.getOperand(Src1Idx
);
3947 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3948 // we need to only have one constant bus use before GFX10.
3949 bool HasImplicitSGPR
= findImplicitSGPRRead(MI
) != AMDGPU::NoRegister
;
3950 if (HasImplicitSGPR
&& ST
.getConstantBusLimit(Opc
) <= 1 &&
3951 Src0
.isReg() && (RI
.isSGPRReg(MRI
, Src0
.getReg()) ||
3952 isLiteralConstantLike(Src0
, InstrDesc
.OpInfo
[Src0Idx
])))
3953 legalizeOpWithMove(MI
, Src0Idx
);
3955 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3956 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3957 // src0/src1 with V_READFIRSTLANE.
3958 if (Opc
== AMDGPU::V_WRITELANE_B32
) {
3959 const DebugLoc
&DL
= MI
.getDebugLoc();
3960 if (Src0
.isReg() && RI
.isVGPR(MRI
, Src0
.getReg())) {
3961 Register Reg
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
3962 BuildMI(*MI
.getParent(), MI
, DL
, get(AMDGPU::V_READFIRSTLANE_B32
), Reg
)
3964 Src0
.ChangeToRegister(Reg
, false);
3966 if (Src1
.isReg() && RI
.isVGPR(MRI
, Src1
.getReg())) {
3967 Register Reg
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
3968 const DebugLoc
&DL
= MI
.getDebugLoc();
3969 BuildMI(*MI
.getParent(), MI
, DL
, get(AMDGPU::V_READFIRSTLANE_B32
), Reg
)
3971 Src1
.ChangeToRegister(Reg
, false);
3976 // No VOP2 instructions support AGPRs.
3977 if (Src0
.isReg() && RI
.isAGPR(MRI
, Src0
.getReg()))
3978 legalizeOpWithMove(MI
, Src0Idx
);
3980 if (Src1
.isReg() && RI
.isAGPR(MRI
, Src1
.getReg()))
3981 legalizeOpWithMove(MI
, Src1Idx
);
3983 // VOP2 src0 instructions support all operand types, so we don't need to check
3984 // their legality. If src1 is already legal, we don't need to do anything.
3985 if (isLegalRegOperand(MRI
, InstrDesc
.OpInfo
[Src1Idx
], Src1
))
3988 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3989 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3990 // select is uniform.
3991 if (Opc
== AMDGPU::V_READLANE_B32
&& Src1
.isReg() &&
3992 RI
.isVGPR(MRI
, Src1
.getReg())) {
3993 Register Reg
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
3994 const DebugLoc
&DL
= MI
.getDebugLoc();
3995 BuildMI(*MI
.getParent(), MI
, DL
, get(AMDGPU::V_READFIRSTLANE_B32
), Reg
)
3997 Src1
.ChangeToRegister(Reg
, false);
4001 // We do not use commuteInstruction here because it is too aggressive and will
4002 // commute if it is possible. We only want to commute here if it improves
4003 // legality. This can be called a fairly large number of times so don't waste
4004 // compile time pointlessly swapping and checking legality again.
4005 if (HasImplicitSGPR
|| !MI
.isCommutable()) {
4006 legalizeOpWithMove(MI
, Src1Idx
);
4010 // If src0 can be used as src1, commuting will make the operands legal.
4011 // Otherwise we have to give up and insert a move.
4013 // TODO: Other immediate-like operand kinds could be commuted if there was a
4014 // MachineOperand::ChangeTo* for them.
4015 if ((!Src1
.isImm() && !Src1
.isReg()) ||
4016 !isLegalRegOperand(MRI
, InstrDesc
.OpInfo
[Src1Idx
], Src0
)) {
4017 legalizeOpWithMove(MI
, Src1Idx
);
4021 int CommutedOpc
= commuteOpcode(MI
);
4022 if (CommutedOpc
== -1) {
4023 legalizeOpWithMove(MI
, Src1Idx
);
4027 MI
.setDesc(get(CommutedOpc
));
4029 Register Src0Reg
= Src0
.getReg();
4030 unsigned Src0SubReg
= Src0
.getSubReg();
4031 bool Src0Kill
= Src0
.isKill();
4034 Src0
.ChangeToImmediate(Src1
.getImm());
4035 else if (Src1
.isReg()) {
4036 Src0
.ChangeToRegister(Src1
.getReg(), false, false, Src1
.isKill());
4037 Src0
.setSubReg(Src1
.getSubReg());
4039 llvm_unreachable("Should only have register or immediate operands");
4041 Src1
.ChangeToRegister(Src0Reg
, false, false, Src0Kill
);
4042 Src1
.setSubReg(Src0SubReg
);
4043 fixImplicitOperands(MI
);
4046 // Legalize VOP3 operands. All operand types are supported for any operand
4047 // but only one literal constant and only starting from GFX10.
4048 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo
&MRI
,
4049 MachineInstr
&MI
) const {
4050 unsigned Opc
= MI
.getOpcode();
4053 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src0
),
4054 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src1
),
4055 AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src2
)
4058 if (Opc
== AMDGPU::V_PERMLANE16_B32
||
4059 Opc
== AMDGPU::V_PERMLANEX16_B32
) {
4060 // src1 and src2 must be scalar
4061 MachineOperand
&Src1
= MI
.getOperand(VOP3Idx
[1]);
4062 MachineOperand
&Src2
= MI
.getOperand(VOP3Idx
[2]);
4063 const DebugLoc
&DL
= MI
.getDebugLoc();
4064 if (Src1
.isReg() && !RI
.isSGPRClass(MRI
.getRegClass(Src1
.getReg()))) {
4065 Register Reg
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
4066 BuildMI(*MI
.getParent(), MI
, DL
, get(AMDGPU::V_READFIRSTLANE_B32
), Reg
)
4068 Src1
.ChangeToRegister(Reg
, false);
4070 if (Src2
.isReg() && !RI
.isSGPRClass(MRI
.getRegClass(Src2
.getReg()))) {
4071 Register Reg
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
4072 BuildMI(*MI
.getParent(), MI
, DL
, get(AMDGPU::V_READFIRSTLANE_B32
), Reg
)
4074 Src2
.ChangeToRegister(Reg
, false);
4078 // Find the one SGPR operand we are allowed to use.
4079 int ConstantBusLimit
= ST
.getConstantBusLimit(Opc
);
4080 int LiteralLimit
= ST
.hasVOP3Literal() ? 1 : 0;
4081 SmallDenseSet
<unsigned> SGPRsUsed
;
4082 unsigned SGPRReg
= findUsedSGPR(MI
, VOP3Idx
);
4083 if (SGPRReg
!= AMDGPU::NoRegister
) {
4084 SGPRsUsed
.insert(SGPRReg
);
4088 for (unsigned i
= 0; i
< 3; ++i
) {
4089 int Idx
= VOP3Idx
[i
];
4092 MachineOperand
&MO
= MI
.getOperand(Idx
);
4095 if (!isLiteralConstantLike(MO
, get(Opc
).OpInfo
[Idx
]))
4098 if (LiteralLimit
> 0 && ConstantBusLimit
> 0) {
4106 legalizeOpWithMove(MI
, Idx
);
4110 if (RI
.hasAGPRs(MRI
.getRegClass(MO
.getReg())) &&
4111 !isOperandLegal(MI
, Idx
, &MO
)) {
4112 legalizeOpWithMove(MI
, Idx
);
4116 if (!RI
.isSGPRClass(MRI
.getRegClass(MO
.getReg())))
4117 continue; // VGPRs are legal
4119 // We can use one SGPR in each VOP3 instruction prior to GFX10
4120 // and two starting from GFX10.
4121 if (SGPRsUsed
.count(MO
.getReg()))
4123 if (ConstantBusLimit
> 0) {
4124 SGPRsUsed
.insert(MO
.getReg());
4129 // If we make it this far, then the operand is not legal and we must
4131 legalizeOpWithMove(MI
, Idx
);
4135 unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg
, MachineInstr
&UseMI
,
4136 MachineRegisterInfo
&MRI
) const {
4137 const TargetRegisterClass
*VRC
= MRI
.getRegClass(SrcReg
);
4138 const TargetRegisterClass
*SRC
= RI
.getEquivalentSGPRClass(VRC
);
4139 Register DstReg
= MRI
.createVirtualRegister(SRC
);
4140 unsigned SubRegs
= RI
.getRegSizeInBits(*VRC
) / 32;
4142 if (RI
.hasAGPRs(VRC
)) {
4143 VRC
= RI
.getEquivalentVGPRClass(VRC
);
4144 Register NewSrcReg
= MRI
.createVirtualRegister(VRC
);
4145 BuildMI(*UseMI
.getParent(), UseMI
, UseMI
.getDebugLoc(),
4146 get(TargetOpcode::COPY
), NewSrcReg
)
4152 BuildMI(*UseMI
.getParent(), UseMI
, UseMI
.getDebugLoc(),
4153 get(AMDGPU::V_READFIRSTLANE_B32
), DstReg
)
4158 SmallVector
<unsigned, 8> SRegs
;
4159 for (unsigned i
= 0; i
< SubRegs
; ++i
) {
4160 Register SGPR
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4161 BuildMI(*UseMI
.getParent(), UseMI
, UseMI
.getDebugLoc(),
4162 get(AMDGPU::V_READFIRSTLANE_B32
), SGPR
)
4163 .addReg(SrcReg
, 0, RI
.getSubRegFromChannel(i
));
4164 SRegs
.push_back(SGPR
);
4167 MachineInstrBuilder MIB
=
4168 BuildMI(*UseMI
.getParent(), UseMI
, UseMI
.getDebugLoc(),
4169 get(AMDGPU::REG_SEQUENCE
), DstReg
);
4170 for (unsigned i
= 0; i
< SubRegs
; ++i
) {
4171 MIB
.addReg(SRegs
[i
]);
4172 MIB
.addImm(RI
.getSubRegFromChannel(i
));
4177 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo
&MRI
,
4178 MachineInstr
&MI
) const {
4180 // If the pointer is store in VGPRs, then we need to move them to
4181 // SGPRs using v_readfirstlane. This is safe because we only select
4182 // loads with uniform pointers to SMRD instruction so we know the
4183 // pointer value is uniform.
4184 MachineOperand
*SBase
= getNamedOperand(MI
, AMDGPU::OpName::sbase
);
4185 if (SBase
&& !RI
.isSGPRClass(MRI
.getRegClass(SBase
->getReg()))) {
4186 unsigned SGPR
= readlaneVGPRToSGPR(SBase
->getReg(), MI
, MRI
);
4187 SBase
->setReg(SGPR
);
4189 MachineOperand
*SOff
= getNamedOperand(MI
, AMDGPU::OpName::soff
);
4190 if (SOff
&& !RI
.isSGPRClass(MRI
.getRegClass(SOff
->getReg()))) {
4191 unsigned SGPR
= readlaneVGPRToSGPR(SOff
->getReg(), MI
, MRI
);
4196 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock
&InsertMBB
,
4197 MachineBasicBlock::iterator I
,
4198 const TargetRegisterClass
*DstRC
,
4200 MachineRegisterInfo
&MRI
,
4201 const DebugLoc
&DL
) const {
4202 Register OpReg
= Op
.getReg();
4203 unsigned OpSubReg
= Op
.getSubReg();
4205 const TargetRegisterClass
*OpRC
= RI
.getSubClassWithSubReg(
4206 RI
.getRegClassForReg(MRI
, OpReg
), OpSubReg
);
4208 // Check if operand is already the correct register class.
4212 Register DstReg
= MRI
.createVirtualRegister(DstRC
);
4213 MachineInstr
*Copy
=
4214 BuildMI(InsertMBB
, I
, DL
, get(AMDGPU::COPY
), DstReg
).add(Op
);
4219 MachineInstr
*Def
= MRI
.getVRegDef(OpReg
);
4223 // Try to eliminate the copy if it is copying an immediate value.
4224 if (Def
->isMoveImmediate())
4225 FoldImmediate(*Copy
, *Def
, OpReg
, &MRI
);
4227 bool ImpDef
= Def
->isImplicitDef();
4228 while (!ImpDef
&& Def
&& Def
->isCopy()) {
4229 Def
= MRI
.getUniqueVRegDef(Def
->getOperand(1).getReg());
4230 ImpDef
= Def
&& Def
->isImplicitDef();
4232 if (!RI
.isSGPRClass(DstRC
) && !Copy
->readsRegister(AMDGPU::EXEC
, &RI
) &&
4234 Copy
->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC
, false, true));
4237 // Emit the actual waterfall loop, executing the wrapped instruction for each
4238 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4239 // iteration, in the worst case we execute 64 (once per lane).
4241 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo
&TII
, MachineRegisterInfo
&MRI
,
4242 MachineBasicBlock
&OrigBB
, MachineBasicBlock
&LoopBB
,
4243 const DebugLoc
&DL
, MachineOperand
&Rsrc
) {
4244 MachineFunction
&MF
= *OrigBB
.getParent();
4245 const GCNSubtarget
&ST
= MF
.getSubtarget
<GCNSubtarget
>();
4246 const SIRegisterInfo
*TRI
= ST
.getRegisterInfo();
4247 unsigned Exec
= ST
.isWave32() ? AMDGPU::EXEC_LO
: AMDGPU::EXEC
;
4248 unsigned SaveExecOpc
=
4249 ST
.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
: AMDGPU::S_AND_SAVEEXEC_B64
;
4250 unsigned XorTermOpc
=
4251 ST
.isWave32() ? AMDGPU::S_XOR_B32_term
: AMDGPU::S_XOR_B64_term
;
4253 ST
.isWave32() ? AMDGPU::S_AND_B32
: AMDGPU::S_AND_B64
;
4254 const auto *BoolXExecRC
= TRI
->getRegClass(AMDGPU::SReg_1_XEXECRegClassID
);
4256 MachineBasicBlock::iterator I
= LoopBB
.begin();
4258 Register VRsrc
= Rsrc
.getReg();
4259 unsigned VRsrcUndef
= getUndefRegState(Rsrc
.isUndef());
4261 Register SaveExec
= MRI
.createVirtualRegister(BoolXExecRC
);
4262 Register CondReg0
= MRI
.createVirtualRegister(BoolXExecRC
);
4263 Register CondReg1
= MRI
.createVirtualRegister(BoolXExecRC
);
4264 Register AndCond
= MRI
.createVirtualRegister(BoolXExecRC
);
4265 Register SRsrcSub0
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4266 Register SRsrcSub1
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4267 Register SRsrcSub2
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4268 Register SRsrcSub3
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4269 Register SRsrc
= MRI
.createVirtualRegister(&AMDGPU::SReg_128RegClass
);
4271 // Beginning of the loop, read the next Rsrc variant.
4272 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_READFIRSTLANE_B32
), SRsrcSub0
)
4273 .addReg(VRsrc
, VRsrcUndef
, AMDGPU::sub0
);
4274 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_READFIRSTLANE_B32
), SRsrcSub1
)
4275 .addReg(VRsrc
, VRsrcUndef
, AMDGPU::sub1
);
4276 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_READFIRSTLANE_B32
), SRsrcSub2
)
4277 .addReg(VRsrc
, VRsrcUndef
, AMDGPU::sub2
);
4278 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_READFIRSTLANE_B32
), SRsrcSub3
)
4279 .addReg(VRsrc
, VRsrcUndef
, AMDGPU::sub3
);
4281 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::REG_SEQUENCE
), SRsrc
)
4283 .addImm(AMDGPU::sub0
)
4285 .addImm(AMDGPU::sub1
)
4287 .addImm(AMDGPU::sub2
)
4289 .addImm(AMDGPU::sub3
);
4291 // Update Rsrc operand to use the SGPR Rsrc.
4293 Rsrc
.setIsKill(true);
4295 // Identify all lanes with identical Rsrc operands in their VGPRs.
4296 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_CMP_EQ_U64_e64
), CondReg0
)
4297 .addReg(SRsrc
, 0, AMDGPU::sub0_sub1
)
4298 .addReg(VRsrc
, 0, AMDGPU::sub0_sub1
);
4299 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::V_CMP_EQ_U64_e64
), CondReg1
)
4300 .addReg(SRsrc
, 0, AMDGPU::sub2_sub3
)
4301 .addReg(VRsrc
, 0, AMDGPU::sub2_sub3
);
4302 BuildMI(LoopBB
, I
, DL
, TII
.get(AndOpc
), AndCond
)
4306 MRI
.setSimpleHint(SaveExec
, AndCond
);
4308 // Update EXEC to matching lanes, saving original to SaveExec.
4309 BuildMI(LoopBB
, I
, DL
, TII
.get(SaveExecOpc
), SaveExec
)
4310 .addReg(AndCond
, RegState::Kill
);
4312 // The original instruction is here; we insert the terminators after it.
4315 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4316 BuildMI(LoopBB
, I
, DL
, TII
.get(XorTermOpc
), Exec
)
4319 BuildMI(LoopBB
, I
, DL
, TII
.get(AMDGPU::S_CBRANCH_EXECNZ
)).addMBB(&LoopBB
);
4322 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4323 // with SGPRs by iterating over all unique values across all lanes.
4324 static void loadSRsrcFromVGPR(const SIInstrInfo
&TII
, MachineInstr
&MI
,
4325 MachineOperand
&Rsrc
, MachineDominatorTree
*MDT
) {
4326 MachineBasicBlock
&MBB
= *MI
.getParent();
4327 MachineFunction
&MF
= *MBB
.getParent();
4328 const GCNSubtarget
&ST
= MF
.getSubtarget
<GCNSubtarget
>();
4329 const SIRegisterInfo
*TRI
= ST
.getRegisterInfo();
4330 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
4331 MachineBasicBlock::iterator
I(&MI
);
4332 const DebugLoc
&DL
= MI
.getDebugLoc();
4333 unsigned Exec
= ST
.isWave32() ? AMDGPU::EXEC_LO
: AMDGPU::EXEC
;
4334 unsigned MovExecOpc
= ST
.isWave32() ? AMDGPU::S_MOV_B32
: AMDGPU::S_MOV_B64
;
4335 const auto *BoolXExecRC
= TRI
->getRegClass(AMDGPU::SReg_1_XEXECRegClassID
);
4337 Register SaveExec
= MRI
.createVirtualRegister(BoolXExecRC
);
4339 // Save the EXEC mask
4340 BuildMI(MBB
, I
, DL
, TII
.get(MovExecOpc
), SaveExec
).addReg(Exec
);
4342 // Killed uses in the instruction we are waterfalling around will be
4343 // incorrect due to the added control-flow.
4344 for (auto &MO
: MI
.uses()) {
4345 if (MO
.isReg() && MO
.isUse()) {
4346 MRI
.clearKillFlags(MO
.getReg());
4350 // To insert the loop we need to split the block. Move everything after this
4351 // point to a new block, and insert a new empty block between the two.
4352 MachineBasicBlock
*LoopBB
= MF
.CreateMachineBasicBlock();
4353 MachineBasicBlock
*RemainderBB
= MF
.CreateMachineBasicBlock();
4354 MachineFunction::iterator
MBBI(MBB
);
4357 MF
.insert(MBBI
, LoopBB
);
4358 MF
.insert(MBBI
, RemainderBB
);
4360 LoopBB
->addSuccessor(LoopBB
);
4361 LoopBB
->addSuccessor(RemainderBB
);
4363 // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4364 MachineBasicBlock::iterator J
= I
++;
4365 RemainderBB
->transferSuccessorsAndUpdatePHIs(&MBB
);
4366 RemainderBB
->splice(RemainderBB
->begin(), &MBB
, I
, MBB
.end());
4367 LoopBB
->splice(LoopBB
->begin(), &MBB
, J
);
4369 MBB
.addSuccessor(LoopBB
);
4371 // Update dominators. We know that MBB immediately dominates LoopBB, that
4372 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4373 // dominates all of the successors transferred to it from MBB that MBB used
4376 MDT
->addNewBlock(LoopBB
, &MBB
);
4377 MDT
->addNewBlock(RemainderBB
, LoopBB
);
4378 for (auto &Succ
: RemainderBB
->successors()) {
4379 if (MDT
->dominates(&MBB
, Succ
)) {
4380 MDT
->changeImmediateDominator(Succ
, RemainderBB
);
4385 emitLoadSRsrcFromVGPRLoop(TII
, MRI
, MBB
, *LoopBB
, DL
, Rsrc
);
4387 // Restore the EXEC mask
4388 MachineBasicBlock::iterator First
= RemainderBB
->begin();
4389 BuildMI(*RemainderBB
, First
, DL
, TII
.get(MovExecOpc
), Exec
).addReg(SaveExec
);
4392 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4393 static std::tuple
<unsigned, unsigned>
4394 extractRsrcPtr(const SIInstrInfo
&TII
, MachineInstr
&MI
, MachineOperand
&Rsrc
) {
4395 MachineBasicBlock
&MBB
= *MI
.getParent();
4396 MachineFunction
&MF
= *MBB
.getParent();
4397 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
4399 // Extract the ptr from the resource descriptor.
4401 TII
.buildExtractSubReg(MI
, MRI
, Rsrc
, &AMDGPU::VReg_128RegClass
,
4402 AMDGPU::sub0_sub1
, &AMDGPU::VReg_64RegClass
);
4404 // Create an empty resource descriptor
4405 Register Zero64
= MRI
.createVirtualRegister(&AMDGPU::SReg_64RegClass
);
4406 Register SRsrcFormatLo
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4407 Register SRsrcFormatHi
= MRI
.createVirtualRegister(&AMDGPU::SGPR_32RegClass
);
4408 Register NewSRsrc
= MRI
.createVirtualRegister(&AMDGPU::SReg_128RegClass
);
4409 uint64_t RsrcDataFormat
= TII
.getDefaultRsrcDataFormat();
4412 BuildMI(MBB
, MI
, MI
.getDebugLoc(), TII
.get(AMDGPU::S_MOV_B64
), Zero64
)
4415 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4416 BuildMI(MBB
, MI
, MI
.getDebugLoc(), TII
.get(AMDGPU::S_MOV_B32
), SRsrcFormatLo
)
4417 .addImm(RsrcDataFormat
& 0xFFFFFFFF);
4419 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4420 BuildMI(MBB
, MI
, MI
.getDebugLoc(), TII
.get(AMDGPU::S_MOV_B32
), SRsrcFormatHi
)
4421 .addImm(RsrcDataFormat
>> 32);
4423 // NewSRsrc = {Zero64, SRsrcFormat}
4424 BuildMI(MBB
, MI
, MI
.getDebugLoc(), TII
.get(AMDGPU::REG_SEQUENCE
), NewSRsrc
)
4426 .addImm(AMDGPU::sub0_sub1
)
4427 .addReg(SRsrcFormatLo
)
4428 .addImm(AMDGPU::sub2
)
4429 .addReg(SRsrcFormatHi
)
4430 .addImm(AMDGPU::sub3
);
4432 return std::make_tuple(RsrcPtr
, NewSRsrc
);
4435 void SIInstrInfo::legalizeOperands(MachineInstr
&MI
,
4436 MachineDominatorTree
*MDT
) const {
4437 MachineFunction
&MF
= *MI
.getParent()->getParent();
4438 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
4441 if (isVOP2(MI
) || isVOPC(MI
)) {
4442 legalizeOperandsVOP2(MRI
, MI
);
4448 legalizeOperandsVOP3(MRI
, MI
);
4454 legalizeOperandsSMRD(MRI
, MI
);
4458 // Legalize REG_SEQUENCE and PHI
4459 // The register class of the operands much be the same type as the register
4460 // class of the output.
4461 if (MI
.getOpcode() == AMDGPU::PHI
) {
4462 const TargetRegisterClass
*RC
= nullptr, *SRC
= nullptr, *VRC
= nullptr;
4463 for (unsigned i
= 1, e
= MI
.getNumOperands(); i
!= e
; i
+= 2) {
4464 if (!MI
.getOperand(i
).isReg() ||
4465 !Register::isVirtualRegister(MI
.getOperand(i
).getReg()))
4467 const TargetRegisterClass
*OpRC
=
4468 MRI
.getRegClass(MI
.getOperand(i
).getReg());
4469 if (RI
.hasVectorRegisters(OpRC
)) {
4476 // If any of the operands are VGPR registers, then they all most be
4477 // otherwise we will create illegal VGPR->SGPR copies when legalizing
4479 if (VRC
|| !RI
.isSGPRClass(getOpRegClass(MI
, 0))) {
4482 VRC
= RI
.hasAGPRs(getOpRegClass(MI
, 0)) ? RI
.getEquivalentAGPRClass(SRC
)
4483 : RI
.getEquivalentVGPRClass(SRC
);
4490 // Update all the operands so they have the same type.
4491 for (unsigned I
= 1, E
= MI
.getNumOperands(); I
!= E
; I
+= 2) {
4492 MachineOperand
&Op
= MI
.getOperand(I
);
4493 if (!Op
.isReg() || !Register::isVirtualRegister(Op
.getReg()))
4496 // MI is a PHI instruction.
4497 MachineBasicBlock
*InsertBB
= MI
.getOperand(I
+ 1).getMBB();
4498 MachineBasicBlock::iterator Insert
= InsertBB
->getFirstTerminator();
4500 // Avoid creating no-op copies with the same src and dst reg class. These
4501 // confuse some of the machine passes.
4502 legalizeGenericOperand(*InsertBB
, Insert
, RC
, Op
, MRI
, MI
.getDebugLoc());
4506 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4507 // VGPR dest type and SGPR sources, insert copies so all operands are
4508 // VGPRs. This seems to help operand folding / the register coalescer.
4509 if (MI
.getOpcode() == AMDGPU::REG_SEQUENCE
) {
4510 MachineBasicBlock
*MBB
= MI
.getParent();
4511 const TargetRegisterClass
*DstRC
= getOpRegClass(MI
, 0);
4512 if (RI
.hasVGPRs(DstRC
)) {
4513 // Update all the operands so they are VGPR register classes. These may
4514 // not be the same register class because REG_SEQUENCE supports mixing
4515 // subregister index types e.g. sub0_sub1 + sub2 + sub3
4516 for (unsigned I
= 1, E
= MI
.getNumOperands(); I
!= E
; I
+= 2) {
4517 MachineOperand
&Op
= MI
.getOperand(I
);
4518 if (!Op
.isReg() || !Register::isVirtualRegister(Op
.getReg()))
4521 const TargetRegisterClass
*OpRC
= MRI
.getRegClass(Op
.getReg());
4522 const TargetRegisterClass
*VRC
= RI
.getEquivalentVGPRClass(OpRC
);
4526 legalizeGenericOperand(*MBB
, MI
, VRC
, Op
, MRI
, MI
.getDebugLoc());
4534 // Legalize INSERT_SUBREG
4535 // src0 must have the same register class as dst
4536 if (MI
.getOpcode() == AMDGPU::INSERT_SUBREG
) {
4537 Register Dst
= MI
.getOperand(0).getReg();
4538 Register Src0
= MI
.getOperand(1).getReg();
4539 const TargetRegisterClass
*DstRC
= MRI
.getRegClass(Dst
);
4540 const TargetRegisterClass
*Src0RC
= MRI
.getRegClass(Src0
);
4541 if (DstRC
!= Src0RC
) {
4542 MachineBasicBlock
*MBB
= MI
.getParent();
4543 MachineOperand
&Op
= MI
.getOperand(1);
4544 legalizeGenericOperand(*MBB
, MI
, DstRC
, Op
, MRI
, MI
.getDebugLoc());
4549 // Legalize SI_INIT_M0
4550 if (MI
.getOpcode() == AMDGPU::SI_INIT_M0
) {
4551 MachineOperand
&Src
= MI
.getOperand(0);
4552 if (Src
.isReg() && RI
.hasVectorRegisters(MRI
.getRegClass(Src
.getReg())))
4553 Src
.setReg(readlaneVGPRToSGPR(Src
.getReg(), MI
, MRI
));
4557 // Legalize MIMG and MUBUF/MTBUF for shaders.
4559 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4560 // scratch memory access. In both cases, the legalization never involves
4561 // conversion to the addr64 form.
4563 (AMDGPU::isShader(MF
.getFunction().getCallingConv()) &&
4564 (isMUBUF(MI
) || isMTBUF(MI
)))) {
4565 MachineOperand
*SRsrc
= getNamedOperand(MI
, AMDGPU::OpName::srsrc
);
4566 if (SRsrc
&& !RI
.isSGPRClass(MRI
.getRegClass(SRsrc
->getReg()))) {
4567 unsigned SGPR
= readlaneVGPRToSGPR(SRsrc
->getReg(), MI
, MRI
);
4568 SRsrc
->setReg(SGPR
);
4571 MachineOperand
*SSamp
= getNamedOperand(MI
, AMDGPU::OpName::ssamp
);
4572 if (SSamp
&& !RI
.isSGPRClass(MRI
.getRegClass(SSamp
->getReg()))) {
4573 unsigned SGPR
= readlaneVGPRToSGPR(SSamp
->getReg(), MI
, MRI
);
4574 SSamp
->setReg(SGPR
);
4579 // Legalize MUBUF* instructions.
4581 AMDGPU::getNamedOperandIdx(MI
.getOpcode(), AMDGPU::OpName::srsrc
);
4582 if (RsrcIdx
!= -1) {
4583 // We have an MUBUF instruction
4584 MachineOperand
*Rsrc
= &MI
.getOperand(RsrcIdx
);
4585 unsigned RsrcRC
= get(MI
.getOpcode()).OpInfo
[RsrcIdx
].RegClass
;
4586 if (RI
.getCommonSubClass(MRI
.getRegClass(Rsrc
->getReg()),
4587 RI
.getRegClass(RsrcRC
))) {
4588 // The operands are legal.
4589 // FIXME: We may need to legalize operands besided srsrc.
4593 // Legalize a VGPR Rsrc.
4595 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4596 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4597 // a zero-value SRsrc.
4599 // If the instruction is _OFFSET (both idxen and offen disabled), and we
4600 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4603 // Otherwise we are on non-ADDR64 hardware, and/or we have
4604 // idxen/offen/bothen and we fall back to a waterfall loop.
4606 MachineBasicBlock
&MBB
= *MI
.getParent();
4608 MachineOperand
*VAddr
= getNamedOperand(MI
, AMDGPU::OpName::vaddr
);
4609 if (VAddr
&& AMDGPU::getIfAddr64Inst(MI
.getOpcode()) != -1) {
4610 // This is already an ADDR64 instruction so we need to add the pointer
4611 // extracted from the resource descriptor to the current value of VAddr.
4612 Register NewVAddrLo
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
4613 Register NewVAddrHi
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
4614 Register NewVAddr
= MRI
.createVirtualRegister(&AMDGPU::VReg_64RegClass
);
4616 const auto *BoolXExecRC
= RI
.getRegClass(AMDGPU::SReg_1_XEXECRegClassID
);
4617 Register CondReg0
= MRI
.createVirtualRegister(BoolXExecRC
);
4618 Register CondReg1
= MRI
.createVirtualRegister(BoolXExecRC
);
4620 unsigned RsrcPtr
, NewSRsrc
;
4621 std::tie(RsrcPtr
, NewSRsrc
) = extractRsrcPtr(*this, MI
, *Rsrc
);
4623 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4624 const DebugLoc
&DL
= MI
.getDebugLoc();
4625 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_ADD_I32_e64
), NewVAddrLo
)
4627 .addReg(RsrcPtr
, 0, AMDGPU::sub0
)
4628 .addReg(VAddr
->getReg(), 0, AMDGPU::sub0
)
4631 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4632 BuildMI(MBB
, MI
, DL
, get(AMDGPU::V_ADDC_U32_e64
), NewVAddrHi
)
4633 .addDef(CondReg1
, RegState::Dead
)
4634 .addReg(RsrcPtr
, 0, AMDGPU::sub1
)
4635 .addReg(VAddr
->getReg(), 0, AMDGPU::sub1
)
4636 .addReg(CondReg0
, RegState::Kill
)
4639 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4640 BuildMI(MBB
, MI
, MI
.getDebugLoc(), get(AMDGPU::REG_SEQUENCE
), NewVAddr
)
4642 .addImm(AMDGPU::sub0
)
4644 .addImm(AMDGPU::sub1
);
4646 VAddr
->setReg(NewVAddr
);
4647 Rsrc
->setReg(NewSRsrc
);
4648 } else if (!VAddr
&& ST
.hasAddr64()) {
4649 // This instructions is the _OFFSET variant, so we need to convert it to
4651 assert(MBB
.getParent()->getSubtarget
<GCNSubtarget
>().getGeneration()
4652 < AMDGPUSubtarget::VOLCANIC_ISLANDS
&&
4653 "FIXME: Need to emit flat atomics here");
4655 unsigned RsrcPtr
, NewSRsrc
;
4656 std::tie(RsrcPtr
, NewSRsrc
) = extractRsrcPtr(*this, MI
, *Rsrc
);
4658 Register NewVAddr
= MRI
.createVirtualRegister(&AMDGPU::VReg_64RegClass
);
4659 MachineOperand
*VData
= getNamedOperand(MI
, AMDGPU::OpName::vdata
);
4660 MachineOperand
*Offset
= getNamedOperand(MI
, AMDGPU::OpName::offset
);
4661 MachineOperand
*SOffset
= getNamedOperand(MI
, AMDGPU::OpName::soffset
);
4662 unsigned Addr64Opcode
= AMDGPU::getAddr64Inst(MI
.getOpcode());
4664 // Atomics rith return have have an additional tied operand and are
4665 // missing some of the special bits.
4666 MachineOperand
*VDataIn
= getNamedOperand(MI
, AMDGPU::OpName::vdata_in
);
4667 MachineInstr
*Addr64
;
4670 // Regular buffer load / store.
4671 MachineInstrBuilder MIB
=
4672 BuildMI(MBB
, MI
, MI
.getDebugLoc(), get(Addr64Opcode
))
4679 // Atomics do not have this operand.
4680 if (const MachineOperand
*GLC
=
4681 getNamedOperand(MI
, AMDGPU::OpName::glc
)) {
4682 MIB
.addImm(GLC
->getImm());
4684 if (const MachineOperand
*DLC
=
4685 getNamedOperand(MI
, AMDGPU::OpName::dlc
)) {
4686 MIB
.addImm(DLC
->getImm());
4689 MIB
.addImm(getNamedImmOperand(MI
, AMDGPU::OpName::slc
));
4691 if (const MachineOperand
*TFE
=
4692 getNamedOperand(MI
, AMDGPU::OpName::tfe
)) {
4693 MIB
.addImm(TFE
->getImm());
4696 MIB
.cloneMemRefs(MI
);
4699 // Atomics with return.
4700 Addr64
= BuildMI(MBB
, MI
, MI
.getDebugLoc(), get(Addr64Opcode
))
4707 .addImm(getNamedImmOperand(MI
, AMDGPU::OpName::slc
))
4711 MI
.removeFromParent();
4713 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4714 BuildMI(MBB
, Addr64
, Addr64
->getDebugLoc(), get(AMDGPU::REG_SEQUENCE
),
4716 .addReg(RsrcPtr
, 0, AMDGPU::sub0
)
4717 .addImm(AMDGPU::sub0
)
4718 .addReg(RsrcPtr
, 0, AMDGPU::sub1
)
4719 .addImm(AMDGPU::sub1
);
4721 // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4723 loadSRsrcFromVGPR(*this, MI
, *Rsrc
, MDT
);
4728 void SIInstrInfo::moveToVALU(MachineInstr
&TopInst
,
4729 MachineDominatorTree
*MDT
) const {
4730 SetVectorType Worklist
;
4731 Worklist
.insert(&TopInst
);
4733 while (!Worklist
.empty()) {
4734 MachineInstr
&Inst
= *Worklist
.pop_back_val();
4735 MachineBasicBlock
*MBB
= Inst
.getParent();
4736 MachineRegisterInfo
&MRI
= MBB
->getParent()->getRegInfo();
4738 unsigned Opcode
= Inst
.getOpcode();
4739 unsigned NewOpcode
= getVALUOp(Inst
);
4741 // Handle some special cases
4745 case AMDGPU::S_ADD_U64_PSEUDO
:
4746 case AMDGPU::S_SUB_U64_PSEUDO
:
4747 splitScalar64BitAddSub(Worklist
, Inst
, MDT
);
4748 Inst
.eraseFromParent();
4750 case AMDGPU::S_ADD_I32
:
4751 case AMDGPU::S_SUB_I32
:
4752 // FIXME: The u32 versions currently selected use the carry.
4753 if (moveScalarAddSub(Worklist
, Inst
, MDT
))
4758 case AMDGPU::S_AND_B64
:
4759 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_AND_B32
, MDT
);
4760 Inst
.eraseFromParent();
4763 case AMDGPU::S_OR_B64
:
4764 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_OR_B32
, MDT
);
4765 Inst
.eraseFromParent();
4768 case AMDGPU::S_XOR_B64
:
4769 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_XOR_B32
, MDT
);
4770 Inst
.eraseFromParent();
4773 case AMDGPU::S_NAND_B64
:
4774 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_NAND_B32
, MDT
);
4775 Inst
.eraseFromParent();
4778 case AMDGPU::S_NOR_B64
:
4779 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_NOR_B32
, MDT
);
4780 Inst
.eraseFromParent();
4783 case AMDGPU::S_XNOR_B64
:
4784 if (ST
.hasDLInsts())
4785 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_XNOR_B32
, MDT
);
4787 splitScalar64BitXnor(Worklist
, Inst
, MDT
);
4788 Inst
.eraseFromParent();
4791 case AMDGPU::S_ANDN2_B64
:
4792 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_ANDN2_B32
, MDT
);
4793 Inst
.eraseFromParent();
4796 case AMDGPU::S_ORN2_B64
:
4797 splitScalar64BitBinaryOp(Worklist
, Inst
, AMDGPU::S_ORN2_B32
, MDT
);
4798 Inst
.eraseFromParent();
4801 case AMDGPU::S_NOT_B64
:
4802 splitScalar64BitUnaryOp(Worklist
, Inst
, AMDGPU::S_NOT_B32
);
4803 Inst
.eraseFromParent();
4806 case AMDGPU::S_BCNT1_I32_B64
:
4807 splitScalar64BitBCNT(Worklist
, Inst
);
4808 Inst
.eraseFromParent();
4811 case AMDGPU::S_BFE_I64
:
4812 splitScalar64BitBFE(Worklist
, Inst
);
4813 Inst
.eraseFromParent();
4816 case AMDGPU::S_LSHL_B32
:
4817 if (ST
.hasOnlyRevVALUShifts()) {
4818 NewOpcode
= AMDGPU::V_LSHLREV_B32_e64
;
4822 case AMDGPU::S_ASHR_I32
:
4823 if (ST
.hasOnlyRevVALUShifts()) {
4824 NewOpcode
= AMDGPU::V_ASHRREV_I32_e64
;
4828 case AMDGPU::S_LSHR_B32
:
4829 if (ST
.hasOnlyRevVALUShifts()) {
4830 NewOpcode
= AMDGPU::V_LSHRREV_B32_e64
;
4834 case AMDGPU::S_LSHL_B64
:
4835 if (ST
.hasOnlyRevVALUShifts()) {
4836 NewOpcode
= AMDGPU::V_LSHLREV_B64
;
4840 case AMDGPU::S_ASHR_I64
:
4841 if (ST
.hasOnlyRevVALUShifts()) {
4842 NewOpcode
= AMDGPU::V_ASHRREV_I64
;
4846 case AMDGPU::S_LSHR_B64
:
4847 if (ST
.hasOnlyRevVALUShifts()) {
4848 NewOpcode
= AMDGPU::V_LSHRREV_B64
;
4853 case AMDGPU::S_ABS_I32
:
4854 lowerScalarAbs(Worklist
, Inst
);
4855 Inst
.eraseFromParent();
4858 case AMDGPU::S_CBRANCH_SCC0
:
4859 case AMDGPU::S_CBRANCH_SCC1
:
4860 // Clear unused bits of vcc
4862 BuildMI(*MBB
, Inst
, Inst
.getDebugLoc(), get(AMDGPU::S_AND_B32
),
4864 .addReg(AMDGPU::EXEC_LO
)
4865 .addReg(AMDGPU::VCC_LO
);
4867 BuildMI(*MBB
, Inst
, Inst
.getDebugLoc(), get(AMDGPU::S_AND_B64
),
4869 .addReg(AMDGPU::EXEC
)
4870 .addReg(AMDGPU::VCC
);
4873 case AMDGPU::S_BFE_U64
:
4874 case AMDGPU::S_BFM_B64
:
4875 llvm_unreachable("Moving this op to VALU not implemented");
4877 case AMDGPU::S_PACK_LL_B32_B16
:
4878 case AMDGPU::S_PACK_LH_B32_B16
:
4879 case AMDGPU::S_PACK_HH_B32_B16
:
4880 movePackToVALU(Worklist
, MRI
, Inst
);
4881 Inst
.eraseFromParent();
4884 case AMDGPU::S_XNOR_B32
:
4885 lowerScalarXnor(Worklist
, Inst
);
4886 Inst
.eraseFromParent();
4889 case AMDGPU::S_NAND_B32
:
4890 splitScalarNotBinop(Worklist
, Inst
, AMDGPU::S_AND_B32
);
4891 Inst
.eraseFromParent();
4894 case AMDGPU::S_NOR_B32
:
4895 splitScalarNotBinop(Worklist
, Inst
, AMDGPU::S_OR_B32
);
4896 Inst
.eraseFromParent();
4899 case AMDGPU::S_ANDN2_B32
:
4900 splitScalarBinOpN2(Worklist
, Inst
, AMDGPU::S_AND_B32
);
4901 Inst
.eraseFromParent();
4904 case AMDGPU::S_ORN2_B32
:
4905 splitScalarBinOpN2(Worklist
, Inst
, AMDGPU::S_OR_B32
);
4906 Inst
.eraseFromParent();
4910 if (NewOpcode
== AMDGPU::INSTRUCTION_LIST_END
) {
4911 // We cannot move this instruction to the VALU, so we should try to
4912 // legalize its operands instead.
4913 legalizeOperands(Inst
, MDT
);
4917 // Use the new VALU Opcode.
4918 const MCInstrDesc
&NewDesc
= get(NewOpcode
);
4919 Inst
.setDesc(NewDesc
);
4921 // Remove any references to SCC. Vector instructions can't read from it, and
4922 // We're just about to add the implicit use / defs of VCC, and we don't want
4924 for (unsigned i
= Inst
.getNumOperands() - 1; i
> 0; --i
) {
4925 MachineOperand
&Op
= Inst
.getOperand(i
);
4926 if (Op
.isReg() && Op
.getReg() == AMDGPU::SCC
) {
4927 // Only propagate through live-def of SCC.
4928 if (Op
.isDef() && !Op
.isDead())
4929 addSCCDefUsersToVALUWorklist(Op
, Inst
, Worklist
);
4930 Inst
.RemoveOperand(i
);
4934 if (Opcode
== AMDGPU::S_SEXT_I32_I8
|| Opcode
== AMDGPU::S_SEXT_I32_I16
) {
4935 // We are converting these to a BFE, so we need to add the missing
4936 // operands for the size and offset.
4937 unsigned Size
= (Opcode
== AMDGPU::S_SEXT_I32_I8
) ? 8 : 16;
4938 Inst
.addOperand(MachineOperand::CreateImm(0));
4939 Inst
.addOperand(MachineOperand::CreateImm(Size
));
4941 } else if (Opcode
== AMDGPU::S_BCNT1_I32_B32
) {
4942 // The VALU version adds the second operand to the result, so insert an
4944 Inst
.addOperand(MachineOperand::CreateImm(0));
4947 Inst
.addImplicitDefUseOperands(*Inst
.getParent()->getParent());
4948 fixImplicitOperands(Inst
);
4950 if (Opcode
== AMDGPU::S_BFE_I32
|| Opcode
== AMDGPU::S_BFE_U32
) {
4951 const MachineOperand
&OffsetWidthOp
= Inst
.getOperand(2);
4952 // If we need to move this to VGPRs, we need to unpack the second operand
4953 // back into the 2 separate ones for bit offset and width.
4954 assert(OffsetWidthOp
.isImm() &&
4955 "Scalar BFE is only implemented for constant width and offset");
4956 uint32_t Imm
= OffsetWidthOp
.getImm();
4958 uint32_t Offset
= Imm
& 0x3f; // Extract bits [5:0].
4959 uint32_t BitWidth
= (Imm
& 0x7f0000) >> 16; // Extract bits [22:16].
4960 Inst
.RemoveOperand(2); // Remove old immediate.
4961 Inst
.addOperand(MachineOperand::CreateImm(Offset
));
4962 Inst
.addOperand(MachineOperand::CreateImm(BitWidth
));
4965 bool HasDst
= Inst
.getOperand(0).isReg() && Inst
.getOperand(0).isDef();
4966 unsigned NewDstReg
= AMDGPU::NoRegister
;
4968 Register DstReg
= Inst
.getOperand(0).getReg();
4969 if (Register::isPhysicalRegister(DstReg
))
4972 // Update the destination register class.
4973 const TargetRegisterClass
*NewDstRC
= getDestEquivalentVGPRClass(Inst
);
4977 if (Inst
.isCopy() &&
4978 Register::isVirtualRegister(Inst
.getOperand(1).getReg()) &&
4979 NewDstRC
== RI
.getRegClassForReg(MRI
, Inst
.getOperand(1).getReg())) {
4980 // Instead of creating a copy where src and dst are the same register
4981 // class, we just replace all uses of dst with src. These kinds of
4982 // copies interfere with the heuristics MachineSink uses to decide
4983 // whether or not to split a critical edge. Since the pass assumes
4984 // that copies will end up as machine instructions and not be
4986 addUsersToMoveToVALUWorklist(DstReg
, MRI
, Worklist
);
4987 MRI
.replaceRegWith(DstReg
, Inst
.getOperand(1).getReg());
4988 MRI
.clearKillFlags(Inst
.getOperand(1).getReg());
4989 Inst
.getOperand(0).setReg(DstReg
);
4991 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4992 // these are deleted later, but at -O0 it would leave a suspicious
4993 // looking illegal copy of an undef register.
4994 for (unsigned I
= Inst
.getNumOperands() - 1; I
!= 0; --I
)
4995 Inst
.RemoveOperand(I
);
4996 Inst
.setDesc(get(AMDGPU::IMPLICIT_DEF
));
5000 NewDstReg
= MRI
.createVirtualRegister(NewDstRC
);
5001 MRI
.replaceRegWith(DstReg
, NewDstReg
);
5004 // Legalize the operands
5005 legalizeOperands(Inst
, MDT
);
5008 addUsersToMoveToVALUWorklist(NewDstReg
, MRI
, Worklist
);
5012 // Add/sub require special handling to deal with carry outs.
5013 bool SIInstrInfo::moveScalarAddSub(SetVectorType
&Worklist
, MachineInstr
&Inst
,
5014 MachineDominatorTree
*MDT
) const {
5015 if (ST
.hasAddNoCarry()) {
5016 // Assume there is no user of scc since we don't select this in that case.
5017 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5020 MachineBasicBlock
&MBB
= *Inst
.getParent();
5021 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5023 Register OldDstReg
= Inst
.getOperand(0).getReg();
5024 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5026 unsigned Opc
= Inst
.getOpcode();
5027 assert(Opc
== AMDGPU::S_ADD_I32
|| Opc
== AMDGPU::S_SUB_I32
);
5029 unsigned NewOpc
= Opc
== AMDGPU::S_ADD_I32
?
5030 AMDGPU::V_ADD_U32_e64
: AMDGPU::V_SUB_U32_e64
;
5032 assert(Inst
.getOperand(3).getReg() == AMDGPU::SCC
);
5033 Inst
.RemoveOperand(3);
5035 Inst
.setDesc(get(NewOpc
));
5036 Inst
.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5037 Inst
.addImplicitDefUseOperands(*MBB
.getParent());
5038 MRI
.replaceRegWith(OldDstReg
, ResultReg
);
5039 legalizeOperands(Inst
, MDT
);
5041 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5048 void SIInstrInfo::lowerScalarAbs(SetVectorType
&Worklist
,
5049 MachineInstr
&Inst
) const {
5050 MachineBasicBlock
&MBB
= *Inst
.getParent();
5051 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5052 MachineBasicBlock::iterator MII
= Inst
;
5053 DebugLoc DL
= Inst
.getDebugLoc();
5055 MachineOperand
&Dest
= Inst
.getOperand(0);
5056 MachineOperand
&Src
= Inst
.getOperand(1);
5057 Register TmpReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5058 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5060 unsigned SubOp
= ST
.hasAddNoCarry() ?
5061 AMDGPU::V_SUB_U32_e32
: AMDGPU::V_SUB_I32_e32
;
5063 BuildMI(MBB
, MII
, DL
, get(SubOp
), TmpReg
)
5065 .addReg(Src
.getReg());
5067 BuildMI(MBB
, MII
, DL
, get(AMDGPU::V_MAX_I32_e64
), ResultReg
)
5068 .addReg(Src
.getReg())
5071 MRI
.replaceRegWith(Dest
.getReg(), ResultReg
);
5072 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5075 void SIInstrInfo::lowerScalarXnor(SetVectorType
&Worklist
,
5076 MachineInstr
&Inst
) const {
5077 MachineBasicBlock
&MBB
= *Inst
.getParent();
5078 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5079 MachineBasicBlock::iterator MII
= Inst
;
5080 const DebugLoc
&DL
= Inst
.getDebugLoc();
5082 MachineOperand
&Dest
= Inst
.getOperand(0);
5083 MachineOperand
&Src0
= Inst
.getOperand(1);
5084 MachineOperand
&Src1
= Inst
.getOperand(2);
5086 if (ST
.hasDLInsts()) {
5087 Register NewDest
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5088 legalizeGenericOperand(MBB
, MII
, &AMDGPU::VGPR_32RegClass
, Src0
, MRI
, DL
);
5089 legalizeGenericOperand(MBB
, MII
, &AMDGPU::VGPR_32RegClass
, Src1
, MRI
, DL
);
5091 BuildMI(MBB
, MII
, DL
, get(AMDGPU::V_XNOR_B32_e64
), NewDest
)
5095 MRI
.replaceRegWith(Dest
.getReg(), NewDest
);
5096 addUsersToMoveToVALUWorklist(NewDest
, MRI
, Worklist
);
5098 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5099 // invert either source and then perform the XOR. If either source is a
5100 // scalar register, then we can leave the inversion on the scalar unit to
5101 // acheive a better distrubution of scalar and vector instructions.
5102 bool Src0IsSGPR
= Src0
.isReg() &&
5103 RI
.isSGPRClass(MRI
.getRegClass(Src0
.getReg()));
5104 bool Src1IsSGPR
= Src1
.isReg() &&
5105 RI
.isSGPRClass(MRI
.getRegClass(Src1
.getReg()));
5107 Register Temp
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5108 Register NewDest
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5110 // Build a pair of scalar instructions and add them to the work list.
5111 // The next iteration over the work list will lower these to the vector
5112 // unit as necessary.
5114 BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B32
), Temp
).add(Src0
);
5115 Xor
= BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_XOR_B32
), NewDest
)
5118 } else if (Src1IsSGPR
) {
5119 BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B32
), Temp
).add(Src1
);
5120 Xor
= BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_XOR_B32
), NewDest
)
5124 Xor
= BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_XOR_B32
), Temp
)
5128 BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B32
), NewDest
).addReg(Temp
);
5129 Worklist
.insert(Not
);
5132 MRI
.replaceRegWith(Dest
.getReg(), NewDest
);
5134 Worklist
.insert(Xor
);
5136 addUsersToMoveToVALUWorklist(NewDest
, MRI
, Worklist
);
5140 void SIInstrInfo::splitScalarNotBinop(SetVectorType
&Worklist
,
5142 unsigned Opcode
) const {
5143 MachineBasicBlock
&MBB
= *Inst
.getParent();
5144 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5145 MachineBasicBlock::iterator MII
= Inst
;
5146 const DebugLoc
&DL
= Inst
.getDebugLoc();
5148 MachineOperand
&Dest
= Inst
.getOperand(0);
5149 MachineOperand
&Src0
= Inst
.getOperand(1);
5150 MachineOperand
&Src1
= Inst
.getOperand(2);
5152 Register NewDest
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5153 Register Interm
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5155 MachineInstr
&Op
= *BuildMI(MBB
, MII
, DL
, get(Opcode
), Interm
)
5159 MachineInstr
&Not
= *BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B32
), NewDest
)
5162 Worklist
.insert(&Op
);
5163 Worklist
.insert(&Not
);
5165 MRI
.replaceRegWith(Dest
.getReg(), NewDest
);
5166 addUsersToMoveToVALUWorklist(NewDest
, MRI
, Worklist
);
5169 void SIInstrInfo::splitScalarBinOpN2(SetVectorType
& Worklist
,
5171 unsigned Opcode
) const {
5172 MachineBasicBlock
&MBB
= *Inst
.getParent();
5173 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5174 MachineBasicBlock::iterator MII
= Inst
;
5175 const DebugLoc
&DL
= Inst
.getDebugLoc();
5177 MachineOperand
&Dest
= Inst
.getOperand(0);
5178 MachineOperand
&Src0
= Inst
.getOperand(1);
5179 MachineOperand
&Src1
= Inst
.getOperand(2);
5181 Register NewDest
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5182 Register Interm
= MRI
.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass
);
5184 MachineInstr
&Not
= *BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B32
), Interm
)
5187 MachineInstr
&Op
= *BuildMI(MBB
, MII
, DL
, get(Opcode
), NewDest
)
5191 Worklist
.insert(&Not
);
5192 Worklist
.insert(&Op
);
5194 MRI
.replaceRegWith(Dest
.getReg(), NewDest
);
5195 addUsersToMoveToVALUWorklist(NewDest
, MRI
, Worklist
);
5198 void SIInstrInfo::splitScalar64BitUnaryOp(
5199 SetVectorType
&Worklist
, MachineInstr
&Inst
,
5200 unsigned Opcode
) const {
5201 MachineBasicBlock
&MBB
= *Inst
.getParent();
5202 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5204 MachineOperand
&Dest
= Inst
.getOperand(0);
5205 MachineOperand
&Src0
= Inst
.getOperand(1);
5206 DebugLoc DL
= Inst
.getDebugLoc();
5208 MachineBasicBlock::iterator MII
= Inst
;
5210 const MCInstrDesc
&InstDesc
= get(Opcode
);
5211 const TargetRegisterClass
*Src0RC
= Src0
.isReg() ?
5212 MRI
.getRegClass(Src0
.getReg()) :
5213 &AMDGPU::SGPR_32RegClass
;
5215 const TargetRegisterClass
*Src0SubRC
= RI
.getSubRegClass(Src0RC
, AMDGPU::sub0
);
5217 MachineOperand SrcReg0Sub0
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5218 AMDGPU::sub0
, Src0SubRC
);
5220 const TargetRegisterClass
*DestRC
= MRI
.getRegClass(Dest
.getReg());
5221 const TargetRegisterClass
*NewDestRC
= RI
.getEquivalentVGPRClass(DestRC
);
5222 const TargetRegisterClass
*NewDestSubRC
= RI
.getSubRegClass(NewDestRC
, AMDGPU::sub0
);
5224 Register DestSub0
= MRI
.createVirtualRegister(NewDestSubRC
);
5225 MachineInstr
&LoHalf
= *BuildMI(MBB
, MII
, DL
, InstDesc
, DestSub0
).add(SrcReg0Sub0
);
5227 MachineOperand SrcReg0Sub1
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5228 AMDGPU::sub1
, Src0SubRC
);
5230 Register DestSub1
= MRI
.createVirtualRegister(NewDestSubRC
);
5231 MachineInstr
&HiHalf
= *BuildMI(MBB
, MII
, DL
, InstDesc
, DestSub1
).add(SrcReg0Sub1
);
5233 Register FullDestReg
= MRI
.createVirtualRegister(NewDestRC
);
5234 BuildMI(MBB
, MII
, DL
, get(TargetOpcode::REG_SEQUENCE
), FullDestReg
)
5236 .addImm(AMDGPU::sub0
)
5238 .addImm(AMDGPU::sub1
);
5240 MRI
.replaceRegWith(Dest
.getReg(), FullDestReg
);
5242 Worklist
.insert(&LoHalf
);
5243 Worklist
.insert(&HiHalf
);
5245 // We don't need to legalizeOperands here because for a single operand, src0
5246 // will support any kind of input.
5248 // Move all users of this moved value.
5249 addUsersToMoveToVALUWorklist(FullDestReg
, MRI
, Worklist
);
5252 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType
&Worklist
,
5254 MachineDominatorTree
*MDT
) const {
5255 bool IsAdd
= (Inst
.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO
);
5257 MachineBasicBlock
&MBB
= *Inst
.getParent();
5258 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5259 const auto *CarryRC
= RI
.getRegClass(AMDGPU::SReg_1_XEXECRegClassID
);
5261 Register FullDestReg
= MRI
.createVirtualRegister(&AMDGPU::VReg_64RegClass
);
5262 Register DestSub0
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5263 Register DestSub1
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5265 Register CarryReg
= MRI
.createVirtualRegister(CarryRC
);
5266 Register DeadCarryReg
= MRI
.createVirtualRegister(CarryRC
);
5268 MachineOperand
&Dest
= Inst
.getOperand(0);
5269 MachineOperand
&Src0
= Inst
.getOperand(1);
5270 MachineOperand
&Src1
= Inst
.getOperand(2);
5271 const DebugLoc
&DL
= Inst
.getDebugLoc();
5272 MachineBasicBlock::iterator MII
= Inst
;
5274 const TargetRegisterClass
*Src0RC
= MRI
.getRegClass(Src0
.getReg());
5275 const TargetRegisterClass
*Src1RC
= MRI
.getRegClass(Src1
.getReg());
5276 const TargetRegisterClass
*Src0SubRC
= RI
.getSubRegClass(Src0RC
, AMDGPU::sub0
);
5277 const TargetRegisterClass
*Src1SubRC
= RI
.getSubRegClass(Src1RC
, AMDGPU::sub0
);
5279 MachineOperand SrcReg0Sub0
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5280 AMDGPU::sub0
, Src0SubRC
);
5281 MachineOperand SrcReg1Sub0
= buildExtractSubRegOrImm(MII
, MRI
, Src1
, Src1RC
,
5282 AMDGPU::sub0
, Src1SubRC
);
5285 MachineOperand SrcReg0Sub1
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5286 AMDGPU::sub1
, Src0SubRC
);
5287 MachineOperand SrcReg1Sub1
= buildExtractSubRegOrImm(MII
, MRI
, Src1
, Src1RC
,
5288 AMDGPU::sub1
, Src1SubRC
);
5290 unsigned LoOpc
= IsAdd
? AMDGPU::V_ADD_I32_e64
: AMDGPU::V_SUB_I32_e64
;
5291 MachineInstr
*LoHalf
=
5292 BuildMI(MBB
, MII
, DL
, get(LoOpc
), DestSub0
)
5293 .addReg(CarryReg
, RegState::Define
)
5296 .addImm(0); // clamp bit
5298 unsigned HiOpc
= IsAdd
? AMDGPU::V_ADDC_U32_e64
: AMDGPU::V_SUBB_U32_e64
;
5299 MachineInstr
*HiHalf
=
5300 BuildMI(MBB
, MII
, DL
, get(HiOpc
), DestSub1
)
5301 .addReg(DeadCarryReg
, RegState::Define
| RegState::Dead
)
5304 .addReg(CarryReg
, RegState::Kill
)
5305 .addImm(0); // clamp bit
5307 BuildMI(MBB
, MII
, DL
, get(TargetOpcode::REG_SEQUENCE
), FullDestReg
)
5309 .addImm(AMDGPU::sub0
)
5311 .addImm(AMDGPU::sub1
);
5313 MRI
.replaceRegWith(Dest
.getReg(), FullDestReg
);
5315 // Try to legalize the operands in case we need to swap the order to keep it
5317 legalizeOperands(*LoHalf
, MDT
);
5318 legalizeOperands(*HiHalf
, MDT
);
5320 // Move all users of this moved vlaue.
5321 addUsersToMoveToVALUWorklist(FullDestReg
, MRI
, Worklist
);
5324 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType
&Worklist
,
5325 MachineInstr
&Inst
, unsigned Opcode
,
5326 MachineDominatorTree
*MDT
) const {
5327 MachineBasicBlock
&MBB
= *Inst
.getParent();
5328 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5330 MachineOperand
&Dest
= Inst
.getOperand(0);
5331 MachineOperand
&Src0
= Inst
.getOperand(1);
5332 MachineOperand
&Src1
= Inst
.getOperand(2);
5333 DebugLoc DL
= Inst
.getDebugLoc();
5335 MachineBasicBlock::iterator MII
= Inst
;
5337 const MCInstrDesc
&InstDesc
= get(Opcode
);
5338 const TargetRegisterClass
*Src0RC
= Src0
.isReg() ?
5339 MRI
.getRegClass(Src0
.getReg()) :
5340 &AMDGPU::SGPR_32RegClass
;
5342 const TargetRegisterClass
*Src0SubRC
= RI
.getSubRegClass(Src0RC
, AMDGPU::sub0
);
5343 const TargetRegisterClass
*Src1RC
= Src1
.isReg() ?
5344 MRI
.getRegClass(Src1
.getReg()) :
5345 &AMDGPU::SGPR_32RegClass
;
5347 const TargetRegisterClass
*Src1SubRC
= RI
.getSubRegClass(Src1RC
, AMDGPU::sub0
);
5349 MachineOperand SrcReg0Sub0
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5350 AMDGPU::sub0
, Src0SubRC
);
5351 MachineOperand SrcReg1Sub0
= buildExtractSubRegOrImm(MII
, MRI
, Src1
, Src1RC
,
5352 AMDGPU::sub0
, Src1SubRC
);
5353 MachineOperand SrcReg0Sub1
= buildExtractSubRegOrImm(MII
, MRI
, Src0
, Src0RC
,
5354 AMDGPU::sub1
, Src0SubRC
);
5355 MachineOperand SrcReg1Sub1
= buildExtractSubRegOrImm(MII
, MRI
, Src1
, Src1RC
,
5356 AMDGPU::sub1
, Src1SubRC
);
5358 const TargetRegisterClass
*DestRC
= MRI
.getRegClass(Dest
.getReg());
5359 const TargetRegisterClass
*NewDestRC
= RI
.getEquivalentVGPRClass(DestRC
);
5360 const TargetRegisterClass
*NewDestSubRC
= RI
.getSubRegClass(NewDestRC
, AMDGPU::sub0
);
5362 Register DestSub0
= MRI
.createVirtualRegister(NewDestSubRC
);
5363 MachineInstr
&LoHalf
= *BuildMI(MBB
, MII
, DL
, InstDesc
, DestSub0
)
5367 Register DestSub1
= MRI
.createVirtualRegister(NewDestSubRC
);
5368 MachineInstr
&HiHalf
= *BuildMI(MBB
, MII
, DL
, InstDesc
, DestSub1
)
5372 Register FullDestReg
= MRI
.createVirtualRegister(NewDestRC
);
5373 BuildMI(MBB
, MII
, DL
, get(TargetOpcode::REG_SEQUENCE
), FullDestReg
)
5375 .addImm(AMDGPU::sub0
)
5377 .addImm(AMDGPU::sub1
);
5379 MRI
.replaceRegWith(Dest
.getReg(), FullDestReg
);
5381 Worklist
.insert(&LoHalf
);
5382 Worklist
.insert(&HiHalf
);
5384 // Move all users of this moved vlaue.
5385 addUsersToMoveToVALUWorklist(FullDestReg
, MRI
, Worklist
);
5388 void SIInstrInfo::splitScalar64BitXnor(SetVectorType
&Worklist
,
5390 MachineDominatorTree
*MDT
) const {
5391 MachineBasicBlock
&MBB
= *Inst
.getParent();
5392 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5394 MachineOperand
&Dest
= Inst
.getOperand(0);
5395 MachineOperand
&Src0
= Inst
.getOperand(1);
5396 MachineOperand
&Src1
= Inst
.getOperand(2);
5397 const DebugLoc
&DL
= Inst
.getDebugLoc();
5399 MachineBasicBlock::iterator MII
= Inst
;
5401 const TargetRegisterClass
*DestRC
= MRI
.getRegClass(Dest
.getReg());
5403 Register Interm
= MRI
.createVirtualRegister(&AMDGPU::SReg_64RegClass
);
5405 MachineOperand
* Op0
;
5406 MachineOperand
* Op1
;
5408 if (Src0
.isReg() && RI
.isSGPRReg(MRI
, Src0
.getReg())) {
5416 BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_NOT_B64
), Interm
)
5419 Register NewDest
= MRI
.createVirtualRegister(DestRC
);
5421 MachineInstr
&Xor
= *BuildMI(MBB
, MII
, DL
, get(AMDGPU::S_XOR_B64
), NewDest
)
5425 MRI
.replaceRegWith(Dest
.getReg(), NewDest
);
5427 Worklist
.insert(&Xor
);
5430 void SIInstrInfo::splitScalar64BitBCNT(
5431 SetVectorType
&Worklist
, MachineInstr
&Inst
) const {
5432 MachineBasicBlock
&MBB
= *Inst
.getParent();
5433 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5435 MachineBasicBlock::iterator MII
= Inst
;
5436 const DebugLoc
&DL
= Inst
.getDebugLoc();
5438 MachineOperand
&Dest
= Inst
.getOperand(0);
5439 MachineOperand
&Src
= Inst
.getOperand(1);
5441 const MCInstrDesc
&InstDesc
= get(AMDGPU::V_BCNT_U32_B32_e64
);
5442 const TargetRegisterClass
*SrcRC
= Src
.isReg() ?
5443 MRI
.getRegClass(Src
.getReg()) :
5444 &AMDGPU::SGPR_32RegClass
;
5446 Register MidReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5447 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5449 const TargetRegisterClass
*SrcSubRC
= RI
.getSubRegClass(SrcRC
, AMDGPU::sub0
);
5451 MachineOperand SrcRegSub0
= buildExtractSubRegOrImm(MII
, MRI
, Src
, SrcRC
,
5452 AMDGPU::sub0
, SrcSubRC
);
5453 MachineOperand SrcRegSub1
= buildExtractSubRegOrImm(MII
, MRI
, Src
, SrcRC
,
5454 AMDGPU::sub1
, SrcSubRC
);
5456 BuildMI(MBB
, MII
, DL
, InstDesc
, MidReg
).add(SrcRegSub0
).addImm(0);
5458 BuildMI(MBB
, MII
, DL
, InstDesc
, ResultReg
).add(SrcRegSub1
).addReg(MidReg
);
5460 MRI
.replaceRegWith(Dest
.getReg(), ResultReg
);
5462 // We don't need to legalize operands here. src0 for etiher instruction can be
5463 // an SGPR, and the second input is unused or determined here.
5464 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5467 void SIInstrInfo::splitScalar64BitBFE(SetVectorType
&Worklist
,
5468 MachineInstr
&Inst
) const {
5469 MachineBasicBlock
&MBB
= *Inst
.getParent();
5470 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
5471 MachineBasicBlock::iterator MII
= Inst
;
5472 const DebugLoc
&DL
= Inst
.getDebugLoc();
5474 MachineOperand
&Dest
= Inst
.getOperand(0);
5475 uint32_t Imm
= Inst
.getOperand(2).getImm();
5476 uint32_t Offset
= Imm
& 0x3f; // Extract bits [5:0].
5477 uint32_t BitWidth
= (Imm
& 0x7f0000) >> 16; // Extract bits [22:16].
5481 // Only sext_inreg cases handled.
5482 assert(Inst
.getOpcode() == AMDGPU::S_BFE_I64
&& BitWidth
<= 32 &&
5483 Offset
== 0 && "Not implemented");
5485 if (BitWidth
< 32) {
5486 Register MidRegLo
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5487 Register MidRegHi
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5488 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VReg_64RegClass
);
5490 BuildMI(MBB
, MII
, DL
, get(AMDGPU::V_BFE_I32
), MidRegLo
)
5491 .addReg(Inst
.getOperand(1).getReg(), 0, AMDGPU::sub0
)
5495 BuildMI(MBB
, MII
, DL
, get(AMDGPU::V_ASHRREV_I32_e32
), MidRegHi
)
5499 BuildMI(MBB
, MII
, DL
, get(TargetOpcode::REG_SEQUENCE
), ResultReg
)
5501 .addImm(AMDGPU::sub0
)
5503 .addImm(AMDGPU::sub1
);
5505 MRI
.replaceRegWith(Dest
.getReg(), ResultReg
);
5506 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5510 MachineOperand
&Src
= Inst
.getOperand(1);
5511 Register TmpReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5512 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VReg_64RegClass
);
5514 BuildMI(MBB
, MII
, DL
, get(AMDGPU::V_ASHRREV_I32_e64
), TmpReg
)
5516 .addReg(Src
.getReg(), 0, AMDGPU::sub0
);
5518 BuildMI(MBB
, MII
, DL
, get(TargetOpcode::REG_SEQUENCE
), ResultReg
)
5519 .addReg(Src
.getReg(), 0, AMDGPU::sub0
)
5520 .addImm(AMDGPU::sub0
)
5522 .addImm(AMDGPU::sub1
);
5524 MRI
.replaceRegWith(Dest
.getReg(), ResultReg
);
5525 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5528 void SIInstrInfo::addUsersToMoveToVALUWorklist(
5530 MachineRegisterInfo
&MRI
,
5531 SetVectorType
&Worklist
) const {
5532 for (MachineRegisterInfo::use_iterator I
= MRI
.use_begin(DstReg
),
5533 E
= MRI
.use_end(); I
!= E
;) {
5534 MachineInstr
&UseMI
= *I
->getParent();
5538 switch (UseMI
.getOpcode()) {
5541 case AMDGPU::SOFT_WQM
:
5543 case AMDGPU::REG_SEQUENCE
:
5545 case AMDGPU::INSERT_SUBREG
:
5548 OpNo
= I
.getOperandNo();
5552 if (!RI
.hasVectorRegisters(getOpRegClass(UseMI
, OpNo
))) {
5553 Worklist
.insert(&UseMI
);
5557 } while (I
!= E
&& I
->getParent() == &UseMI
);
5564 void SIInstrInfo::movePackToVALU(SetVectorType
&Worklist
,
5565 MachineRegisterInfo
&MRI
,
5566 MachineInstr
&Inst
) const {
5567 Register ResultReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5568 MachineBasicBlock
*MBB
= Inst
.getParent();
5569 MachineOperand
&Src0
= Inst
.getOperand(1);
5570 MachineOperand
&Src1
= Inst
.getOperand(2);
5571 const DebugLoc
&DL
= Inst
.getDebugLoc();
5573 switch (Inst
.getOpcode()) {
5574 case AMDGPU::S_PACK_LL_B32_B16
: {
5575 Register ImmReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5576 Register TmpReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5578 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
5580 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_MOV_B32_e32
), ImmReg
)
5583 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_AND_B32_e64
), TmpReg
)
5584 .addReg(ImmReg
, RegState::Kill
)
5587 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_LSHL_OR_B32
), ResultReg
)
5590 .addReg(TmpReg
, RegState::Kill
);
5593 case AMDGPU::S_PACK_LH_B32_B16
: {
5594 Register ImmReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5595 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_MOV_B32_e32
), ImmReg
)
5597 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_BFI_B32
), ResultReg
)
5598 .addReg(ImmReg
, RegState::Kill
)
5603 case AMDGPU::S_PACK_HH_B32_B16
: {
5604 Register ImmReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5605 Register TmpReg
= MRI
.createVirtualRegister(&AMDGPU::VGPR_32RegClass
);
5606 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_LSHRREV_B32_e64
), TmpReg
)
5609 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_MOV_B32_e32
), ImmReg
)
5610 .addImm(0xffff0000);
5611 BuildMI(*MBB
, Inst
, DL
, get(AMDGPU::V_AND_OR_B32
), ResultReg
)
5613 .addReg(ImmReg
, RegState::Kill
)
5614 .addReg(TmpReg
, RegState::Kill
);
5618 llvm_unreachable("unhandled s_pack_* instruction");
5621 MachineOperand
&Dest
= Inst
.getOperand(0);
5622 MRI
.replaceRegWith(Dest
.getReg(), ResultReg
);
5623 addUsersToMoveToVALUWorklist(ResultReg
, MRI
, Worklist
);
5626 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand
&Op
,
5627 MachineInstr
&SCCDefInst
,
5628 SetVectorType
&Worklist
) const {
5629 // Ensure that def inst defines SCC, which is still live.
5630 assert(Op
.isReg() && Op
.getReg() == AMDGPU::SCC
&& Op
.isDef() &&
5631 !Op
.isDead() && Op
.getParent() == &SCCDefInst
);
5632 // This assumes that all the users of SCC are in the same block
5634 for (MachineInstr
&MI
: // Skip the def inst itself.
5635 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst
)),
5636 SCCDefInst
.getParent()->end())) {
5637 // Check if SCC is used first.
5638 if (MI
.findRegisterUseOperandIdx(AMDGPU::SCC
, false, &RI
) != -1)
5639 Worklist
.insert(&MI
);
5640 // Exit if we find another SCC def.
5641 if (MI
.findRegisterDefOperandIdx(AMDGPU::SCC
, false, false, &RI
) != -1)
5646 const TargetRegisterClass
*SIInstrInfo::getDestEquivalentVGPRClass(
5647 const MachineInstr
&Inst
) const {
5648 const TargetRegisterClass
*NewDstRC
= getOpRegClass(Inst
, 0);
5650 switch (Inst
.getOpcode()) {
5651 // For target instructions, getOpRegClass just returns the virtual register
5652 // class associated with the operand, so we need to find an equivalent VGPR
5653 // register class in order to move the instruction to the VALU.
5656 case AMDGPU::REG_SEQUENCE
:
5657 case AMDGPU::INSERT_SUBREG
:
5659 case AMDGPU::SOFT_WQM
:
5661 const TargetRegisterClass
*SrcRC
= getOpRegClass(Inst
, 1);
5662 if (RI
.hasAGPRs(SrcRC
)) {
5663 if (RI
.hasAGPRs(NewDstRC
))
5666 NewDstRC
= RI
.getEquivalentAGPRClass(NewDstRC
);
5670 if (RI
.hasVGPRs(NewDstRC
))
5673 NewDstRC
= RI
.getEquivalentVGPRClass(NewDstRC
);
5685 // Find the one SGPR operand we are allowed to use.
5686 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr
&MI
,
5687 int OpIndices
[3]) const {
5688 const MCInstrDesc
&Desc
= MI
.getDesc();
5690 // Find the one SGPR operand we are allowed to use.
5692 // First we need to consider the instruction's operand requirements before
5693 // legalizing. Some operands are required to be SGPRs, such as implicit uses
5694 // of VCC, but we are still bound by the constant bus requirement to only use
5697 // If the operand's class is an SGPR, we can never move it.
5699 unsigned SGPRReg
= findImplicitSGPRRead(MI
);
5700 if (SGPRReg
!= AMDGPU::NoRegister
)
5703 unsigned UsedSGPRs
[3] = { AMDGPU::NoRegister
};
5704 const MachineRegisterInfo
&MRI
= MI
.getParent()->getParent()->getRegInfo();
5706 for (unsigned i
= 0; i
< 3; ++i
) {
5707 int Idx
= OpIndices
[i
];
5711 const MachineOperand
&MO
= MI
.getOperand(Idx
);
5715 // Is this operand statically required to be an SGPR based on the operand
5717 const TargetRegisterClass
*OpRC
= RI
.getRegClass(Desc
.OpInfo
[Idx
].RegClass
);
5718 bool IsRequiredSGPR
= RI
.isSGPRClass(OpRC
);
5722 // If this could be a VGPR or an SGPR, Check the dynamic register class.
5723 Register Reg
= MO
.getReg();
5724 const TargetRegisterClass
*RegRC
= MRI
.getRegClass(Reg
);
5725 if (RI
.isSGPRClass(RegRC
))
5729 // We don't have a required SGPR operand, so we have a bit more freedom in
5730 // selecting operands to move.
5732 // Try to select the most used SGPR. If an SGPR is equal to one of the
5733 // others, we choose that.
5736 // V_FMA_F32 v0, s0, s0, s0 -> No moves
5737 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
5739 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
5742 if (UsedSGPRs
[0] != AMDGPU::NoRegister
) {
5743 if (UsedSGPRs
[0] == UsedSGPRs
[1] || UsedSGPRs
[0] == UsedSGPRs
[2])
5744 SGPRReg
= UsedSGPRs
[0];
5747 if (SGPRReg
== AMDGPU::NoRegister
&& UsedSGPRs
[1] != AMDGPU::NoRegister
) {
5748 if (UsedSGPRs
[1] == UsedSGPRs
[2])
5749 SGPRReg
= UsedSGPRs
[1];
5755 MachineOperand
*SIInstrInfo::getNamedOperand(MachineInstr
&MI
,
5756 unsigned OperandName
) const {
5757 int Idx
= AMDGPU::getNamedOperandIdx(MI
.getOpcode(), OperandName
);
5761 return &MI
.getOperand(Idx
);
5764 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
5765 if (ST
.getGeneration() >= AMDGPUSubtarget::GFX10
) {
5766 return (22ULL << 44) | // IMG_FORMAT_32_FLOAT
5767 (1ULL << 56) | // RESOURCE_LEVEL = 1
5768 (3ULL << 60); // OOB_SELECT = 3
5771 uint64_t RsrcDataFormat
= AMDGPU::RSRC_DATA_FORMAT
;
5772 if (ST
.isAmdHsaOS()) {
5773 // Set ATC = 1. GFX9 doesn't have this bit.
5774 if (ST
.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS
)
5775 RsrcDataFormat
|= (1ULL << 56);
5777 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
5778 // BTW, it disables TC L2 and therefore decreases performance.
5779 if (ST
.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS
)
5780 RsrcDataFormat
|= (2ULL << 59);
5783 return RsrcDataFormat
;
5786 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
5787 uint64_t Rsrc23
= getDefaultRsrcDataFormat() |
5788 AMDGPU::RSRC_TID_ENABLE
|
5789 0xffffffff; // Size;
5791 // GFX9 doesn't have ELEMENT_SIZE.
5792 if (ST
.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS
) {
5793 uint64_t EltSizeValue
= Log2_32(ST
.getMaxPrivateElementSize()) - 1;
5794 Rsrc23
|= EltSizeValue
<< AMDGPU::RSRC_ELEMENT_SIZE_SHIFT
;
5797 // IndexStride = 64 / 32.
5798 uint64_t IndexStride
= ST
.getWavefrontSize() == 64 ? 3 : 2;
5799 Rsrc23
|= IndexStride
<< AMDGPU::RSRC_INDEX_STRIDE_SHIFT
;
5801 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
5802 // Clear them unless we want a huge stride.
5803 if (ST
.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS
&&
5804 ST
.getGeneration() <= AMDGPUSubtarget::GFX9
)
5805 Rsrc23
&= ~AMDGPU::RSRC_DATA_FORMAT
;
5810 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr
&MI
) const {
5811 unsigned Opc
= MI
.getOpcode();
5816 bool SIInstrInfo::isHighLatencyInstruction(const MachineInstr
&MI
) const {
5817 unsigned Opc
= MI
.getOpcode();
5819 return isMUBUF(Opc
) || isMTBUF(Opc
) || isMIMG(Opc
);
5822 unsigned SIInstrInfo::isStackAccess(const MachineInstr
&MI
,
5823 int &FrameIndex
) const {
5824 const MachineOperand
*Addr
= getNamedOperand(MI
, AMDGPU::OpName::vaddr
);
5825 if (!Addr
|| !Addr
->isFI())
5826 return AMDGPU::NoRegister
;
5828 assert(!MI
.memoperands_empty() &&
5829 (*MI
.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS
);
5831 FrameIndex
= Addr
->getIndex();
5832 return getNamedOperand(MI
, AMDGPU::OpName::vdata
)->getReg();
5835 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr
&MI
,
5836 int &FrameIndex
) const {
5837 const MachineOperand
*Addr
= getNamedOperand(MI
, AMDGPU::OpName::addr
);
5838 assert(Addr
&& Addr
->isFI());
5839 FrameIndex
= Addr
->getIndex();
5840 return getNamedOperand(MI
, AMDGPU::OpName::data
)->getReg();
5843 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr
&MI
,
5844 int &FrameIndex
) const {
5846 return AMDGPU::NoRegister
;
5848 if (isMUBUF(MI
) || isVGPRSpill(MI
))
5849 return isStackAccess(MI
, FrameIndex
);
5851 if (isSGPRSpill(MI
))
5852 return isSGPRStackAccess(MI
, FrameIndex
);
5854 return AMDGPU::NoRegister
;
5857 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr
&MI
,
5858 int &FrameIndex
) const {
5860 return AMDGPU::NoRegister
;
5862 if (isMUBUF(MI
) || isVGPRSpill(MI
))
5863 return isStackAccess(MI
, FrameIndex
);
5865 if (isSGPRSpill(MI
))
5866 return isSGPRStackAccess(MI
, FrameIndex
);
5868 return AMDGPU::NoRegister
;
5871 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr
&MI
) const {
5873 MachineBasicBlock::const_instr_iterator I
= MI
.getIterator();
5874 MachineBasicBlock::const_instr_iterator E
= MI
.getParent()->instr_end();
5875 while (++I
!= E
&& I
->isInsideBundle()) {
5876 assert(!I
->isBundle() && "No nested bundle!");
5877 Size
+= getInstSizeInBytes(*I
);
5883 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr
&MI
) const {
5884 unsigned Opc
= MI
.getOpcode();
5885 const MCInstrDesc
&Desc
= getMCOpcodeFromPseudo(Opc
);
5886 unsigned DescSize
= Desc
.getSize();
5888 // If we have a definitive size, we can use it. Otherwise we need to inspect
5889 // the operands to know the size.
5890 if (isFixedSize(MI
))
5893 // 4-byte instructions may have a 32-bit literal encoded after them. Check
5894 // operands that coud ever be literals.
5895 if (isVALU(MI
) || isSALU(MI
)) {
5896 int Src0Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src0
);
5898 return DescSize
; // No operands.
5900 if (isLiteralConstantLike(MI
.getOperand(Src0Idx
), Desc
.OpInfo
[Src0Idx
]))
5901 return isVOP3(MI
) ? 12 : (DescSize
+ 4);
5903 int Src1Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src1
);
5907 if (isLiteralConstantLike(MI
.getOperand(Src1Idx
), Desc
.OpInfo
[Src1Idx
]))
5908 return isVOP3(MI
) ? 12 : (DescSize
+ 4);
5910 int Src2Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::src2
);
5914 if (isLiteralConstantLike(MI
.getOperand(Src2Idx
), Desc
.OpInfo
[Src2Idx
]))
5915 return isVOP3(MI
) ? 12 : (DescSize
+ 4);
5920 // Check whether we have extra NSA words.
5922 int VAddr0Idx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::vaddr0
);
5926 int RSrcIdx
= AMDGPU::getNamedOperandIdx(Opc
, AMDGPU::OpName::srsrc
);
5927 return 8 + 4 * ((RSrcIdx
- VAddr0Idx
+ 2) / 4);
5931 case TargetOpcode::IMPLICIT_DEF
:
5932 case TargetOpcode::KILL
:
5933 case TargetOpcode::DBG_VALUE
:
5934 case TargetOpcode::EH_LABEL
:
5936 case TargetOpcode::BUNDLE
:
5937 return getInstBundleSize(MI
);
5938 case TargetOpcode::INLINEASM
:
5939 case TargetOpcode::INLINEASM_BR
: {
5940 const MachineFunction
*MF
= MI
.getParent()->getParent();
5941 const char *AsmStr
= MI
.getOperand(0).getSymbolName();
5942 return getInlineAsmLength(AsmStr
, *MF
->getTarget().getMCAsmInfo(),
5943 &MF
->getSubtarget());
5950 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr
&MI
) const {
5954 if (MI
.memoperands_empty())
5957 for (const MachineMemOperand
*MMO
: MI
.memoperands()) {
5958 if (MMO
->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS
)
5964 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr
&Branch
) const {
5965 return Branch
.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO
;
5968 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock
*IfEntry
,
5969 MachineBasicBlock
*IfEnd
) const {
5970 MachineBasicBlock::iterator TI
= IfEntry
->getFirstTerminator();
5971 assert(TI
!= IfEntry
->end());
5973 MachineInstr
*Branch
= &(*TI
);
5974 MachineFunction
*MF
= IfEntry
->getParent();
5975 MachineRegisterInfo
&MRI
= IfEntry
->getParent()->getRegInfo();
5977 if (Branch
->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO
) {
5978 Register DstReg
= MRI
.createVirtualRegister(RI
.getBoolRC());
5979 MachineInstr
*SIIF
=
5980 BuildMI(*MF
, Branch
->getDebugLoc(), get(AMDGPU::SI_IF
), DstReg
)
5981 .add(Branch
->getOperand(0))
5982 .add(Branch
->getOperand(1));
5983 MachineInstr
*SIEND
=
5984 BuildMI(*MF
, Branch
->getDebugLoc(), get(AMDGPU::SI_END_CF
))
5988 IfEntry
->insert(IfEntry
->end(), SIIF
);
5989 IfEnd
->insert(IfEnd
->getFirstNonPHI(), SIEND
);
5993 void SIInstrInfo::convertNonUniformLoopRegion(
5994 MachineBasicBlock
*LoopEntry
, MachineBasicBlock
*LoopEnd
) const {
5995 MachineBasicBlock::iterator TI
= LoopEnd
->getFirstTerminator();
5996 // We expect 2 terminators, one conditional and one unconditional.
5997 assert(TI
!= LoopEnd
->end());
5999 MachineInstr
*Branch
= &(*TI
);
6000 MachineFunction
*MF
= LoopEnd
->getParent();
6001 MachineRegisterInfo
&MRI
= LoopEnd
->getParent()->getRegInfo();
6003 if (Branch
->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO
) {
6005 Register DstReg
= MRI
.createVirtualRegister(RI
.getBoolRC());
6006 Register BackEdgeReg
= MRI
.createVirtualRegister(RI
.getBoolRC());
6007 MachineInstrBuilder HeaderPHIBuilder
=
6008 BuildMI(*(MF
), Branch
->getDebugLoc(), get(TargetOpcode::PHI
), DstReg
);
6009 for (MachineBasicBlock::pred_iterator PI
= LoopEntry
->pred_begin(),
6010 E
= LoopEntry
->pred_end();
6012 if (*PI
== LoopEnd
) {
6013 HeaderPHIBuilder
.addReg(BackEdgeReg
);
6015 MachineBasicBlock
*PMBB
= *PI
;
6016 Register ZeroReg
= MRI
.createVirtualRegister(RI
.getBoolRC());
6017 materializeImmediate(*PMBB
, PMBB
->getFirstTerminator(), DebugLoc(),
6019 HeaderPHIBuilder
.addReg(ZeroReg
);
6021 HeaderPHIBuilder
.addMBB(*PI
);
6023 MachineInstr
*HeaderPhi
= HeaderPHIBuilder
;
6024 MachineInstr
*SIIFBREAK
= BuildMI(*(MF
), Branch
->getDebugLoc(),
6025 get(AMDGPU::SI_IF_BREAK
), BackEdgeReg
)
6027 .add(Branch
->getOperand(0));
6028 MachineInstr
*SILOOP
=
6029 BuildMI(*(MF
), Branch
->getDebugLoc(), get(AMDGPU::SI_LOOP
))
6030 .addReg(BackEdgeReg
)
6033 LoopEntry
->insert(LoopEntry
->begin(), HeaderPhi
);
6035 LoopEnd
->insert(LoopEnd
->end(), SIIFBREAK
);
6036 LoopEnd
->insert(LoopEnd
->end(), SILOOP
);
6040 ArrayRef
<std::pair
<int, const char *>>
6041 SIInstrInfo::getSerializableTargetIndices() const {
6042 static const std::pair
<int, const char *> TargetIndices
[] = {
6043 {AMDGPU::TI_CONSTDATA_START
, "amdgpu-constdata-start"},
6044 {AMDGPU::TI_SCRATCH_RSRC_DWORD0
, "amdgpu-scratch-rsrc-dword0"},
6045 {AMDGPU::TI_SCRATCH_RSRC_DWORD1
, "amdgpu-scratch-rsrc-dword1"},
6046 {AMDGPU::TI_SCRATCH_RSRC_DWORD2
, "amdgpu-scratch-rsrc-dword2"},
6047 {AMDGPU::TI_SCRATCH_RSRC_DWORD3
, "amdgpu-scratch-rsrc-dword3"}};
6048 return makeArrayRef(TargetIndices
);
6051 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
6052 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
6053 ScheduleHazardRecognizer
*
6054 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData
*II
,
6055 const ScheduleDAG
*DAG
) const {
6056 return new GCNHazardRecognizer(DAG
->MF
);
6059 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
6061 ScheduleHazardRecognizer
*
6062 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction
&MF
) const {
6063 return new GCNHazardRecognizer(MF
);
6066 std::pair
<unsigned, unsigned>
6067 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF
) const {
6068 return std::make_pair(TF
& MO_MASK
, TF
& ~MO_MASK
);
6071 ArrayRef
<std::pair
<unsigned, const char *>>
6072 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
6073 static const std::pair
<unsigned, const char *> TargetFlags
[] = {
6074 { MO_GOTPCREL
, "amdgpu-gotprel" },
6075 { MO_GOTPCREL32_LO
, "amdgpu-gotprel32-lo" },
6076 { MO_GOTPCREL32_HI
, "amdgpu-gotprel32-hi" },
6077 { MO_REL32_LO
, "amdgpu-rel32-lo" },
6078 { MO_REL32_HI
, "amdgpu-rel32-hi" },
6079 { MO_ABS32_LO
, "amdgpu-abs32-lo" },
6080 { MO_ABS32_HI
, "amdgpu-abs32-hi" },
6083 return makeArrayRef(TargetFlags
);
6086 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr
&MI
) const {
6087 return !MI
.isTerminator() && MI
.getOpcode() != AMDGPU::COPY
&&
6088 MI
.modifiesRegister(AMDGPU::EXEC
, &RI
);
6092 SIInstrInfo::getAddNoCarry(MachineBasicBlock
&MBB
,
6093 MachineBasicBlock::iterator I
,
6095 unsigned DestReg
) const {
6096 if (ST
.hasAddNoCarry())
6097 return BuildMI(MBB
, I
, DL
, get(AMDGPU::V_ADD_U32_e64
), DestReg
);
6099 MachineRegisterInfo
&MRI
= MBB
.getParent()->getRegInfo();
6100 Register UnusedCarry
= MRI
.createVirtualRegister(RI
.getBoolRC());
6101 MRI
.setRegAllocationHint(UnusedCarry
, 0, RI
.getVCC());
6103 return BuildMI(MBB
, I
, DL
, get(AMDGPU::V_ADD_I32_e64
), DestReg
)
6104 .addReg(UnusedCarry
, RegState::Define
| RegState::Dead
);
6107 MachineInstrBuilder
SIInstrInfo::getAddNoCarry(MachineBasicBlock
&MBB
,
6108 MachineBasicBlock::iterator I
,
6111 RegScavenger
&RS
) const {
6112 if (ST
.hasAddNoCarry())
6113 return BuildMI(MBB
, I
, DL
, get(AMDGPU::V_ADD_U32_e32
), DestReg
);
6115 Register UnusedCarry
= RS
.scavengeRegister(RI
.getBoolRC(), I
, 0, false);
6116 // TODO: Users need to deal with this.
6117 if (!UnusedCarry
.isValid())
6118 return MachineInstrBuilder();
6120 return BuildMI(MBB
, I
, DL
, get(AMDGPU::V_ADD_I32_e64
), DestReg
)
6121 .addReg(UnusedCarry
, RegState::Define
| RegState::Dead
);
6124 bool SIInstrInfo::isKillTerminator(unsigned Opcode
) {
6126 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR
:
6127 case AMDGPU::SI_KILL_I1_TERMINATOR
:
6134 const MCInstrDesc
&SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode
) const {
6136 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO
:
6137 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR
);
6138 case AMDGPU::SI_KILL_I1_PSEUDO
:
6139 return get(AMDGPU::SI_KILL_I1_TERMINATOR
);
6141 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
6145 void SIInstrInfo::fixImplicitOperands(MachineInstr
&MI
) const {
6146 MachineBasicBlock
*MBB
= MI
.getParent();
6147 MachineFunction
*MF
= MBB
->getParent();
6148 const GCNSubtarget
&ST
= MF
->getSubtarget
<GCNSubtarget
>();
6153 for (auto &Op
: MI
.implicit_operands()) {
6154 if (Op
.isReg() && Op
.getReg() == AMDGPU::VCC
)
6155 Op
.setReg(AMDGPU::VCC_LO
);
6159 bool SIInstrInfo::isBufferSMRD(const MachineInstr
&MI
) const {
6163 // Check that it is using a buffer resource.
6164 int Idx
= AMDGPU::getNamedOperandIdx(MI
.getOpcode(), AMDGPU::OpName::sbase
);
6165 if (Idx
== -1) // e.g. s_memtime
6168 const auto RCID
= MI
.getDesc().OpInfo
[Idx
].RegClass
;
6169 return RCID
== AMDGPU::SReg_128RegClassID
;
6172 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset
, unsigned AddrSpace
,
6173 bool Signed
) const {
6174 // TODO: Should 0 be special cased?
6175 if (!ST
.hasFlatInstOffsets())
6178 if (ST
.hasFlatSegmentOffsetBug() && AddrSpace
== AMDGPUAS::FLAT_ADDRESS
)
6181 if (ST
.getGeneration() >= AMDGPUSubtarget::GFX10
) {
6182 return (Signed
&& isInt
<12>(Offset
)) ||
6183 (!Signed
&& isUInt
<11>(Offset
));
6186 return (Signed
&& isInt
<13>(Offset
)) ||
6187 (!Signed
&& isUInt
<12>(Offset
));
6191 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
6192 enum SIEncodingFamily
{
6203 static SIEncodingFamily
subtargetEncodingFamily(const GCNSubtarget
&ST
) {
6204 switch (ST
.getGeneration()) {
6207 case AMDGPUSubtarget::SOUTHERN_ISLANDS
:
6208 case AMDGPUSubtarget::SEA_ISLANDS
:
6209 return SIEncodingFamily::SI
;
6210 case AMDGPUSubtarget::VOLCANIC_ISLANDS
:
6211 case AMDGPUSubtarget::GFX9
:
6212 return SIEncodingFamily::VI
;
6213 case AMDGPUSubtarget::GFX10
:
6214 return SIEncodingFamily::GFX10
;
6216 llvm_unreachable("Unknown subtarget generation!");
6219 int SIInstrInfo::pseudoToMCOpcode(int Opcode
) const {
6220 SIEncodingFamily Gen
= subtargetEncodingFamily(ST
);
6222 if ((get(Opcode
).TSFlags
& SIInstrFlags::renamedInGFX9
) != 0 &&
6223 ST
.getGeneration() == AMDGPUSubtarget::GFX9
)
6224 Gen
= SIEncodingFamily::GFX9
;
6226 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
6227 // subtarget has UnpackedD16VMem feature.
6228 // TODO: remove this when we discard GFX80 encoding.
6229 if (ST
.hasUnpackedD16VMem() && (get(Opcode
).TSFlags
& SIInstrFlags::D16Buf
))
6230 Gen
= SIEncodingFamily::GFX80
;
6232 if (get(Opcode
).TSFlags
& SIInstrFlags::SDWA
) {
6233 switch (ST
.getGeneration()) {
6235 Gen
= SIEncodingFamily::SDWA
;
6237 case AMDGPUSubtarget::GFX9
:
6238 Gen
= SIEncodingFamily::SDWA9
;
6240 case AMDGPUSubtarget::GFX10
:
6241 Gen
= SIEncodingFamily::SDWA10
;
6246 int MCOp
= AMDGPU::getMCOpcode(Opcode
, Gen
);
6248 // -1 means that Opcode is already a native instruction.
6252 // (uint16_t)-1 means that Opcode is a pseudo instruction that has
6253 // no encoding in the given subtarget generation.
6254 if (MCOp
== (uint16_t)-1)
6261 TargetInstrInfo::RegSubRegPair
getRegOrUndef(const MachineOperand
&RegOpnd
) {
6262 assert(RegOpnd
.isReg());
6263 return RegOpnd
.isUndef() ? TargetInstrInfo::RegSubRegPair() :
6264 getRegSubRegPair(RegOpnd
);
6267 TargetInstrInfo::RegSubRegPair
6268 llvm::getRegSequenceSubReg(MachineInstr
&MI
, unsigned SubReg
) {
6269 assert(MI
.isRegSequence());
6270 for (unsigned I
= 0, E
= (MI
.getNumOperands() - 1)/ 2; I
< E
; ++I
)
6271 if (MI
.getOperand(1 + 2 * I
+ 1).getImm() == SubReg
) {
6272 auto &RegOp
= MI
.getOperand(1 + 2 * I
);
6273 return getRegOrUndef(RegOp
);
6275 return TargetInstrInfo::RegSubRegPair();
6278 // Try to find the definition of reg:subreg in subreg-manipulation pseudos
6279 // Following a subreg of reg:subreg isn't supported
6280 static bool followSubRegDef(MachineInstr
&MI
,
6281 TargetInstrInfo::RegSubRegPair
&RSR
) {
6284 switch (MI
.getOpcode()) {
6286 case AMDGPU::REG_SEQUENCE
:
6287 RSR
= getRegSequenceSubReg(MI
, RSR
.SubReg
);
6289 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
6290 case AMDGPU::INSERT_SUBREG
:
6291 if (RSR
.SubReg
== (unsigned)MI
.getOperand(3).getImm())
6292 // inserted the subreg we're looking for
6293 RSR
= getRegOrUndef(MI
.getOperand(2));
6294 else { // the subreg in the rest of the reg
6295 auto R1
= getRegOrUndef(MI
.getOperand(1));
6296 if (R1
.SubReg
) // subreg of subreg isn't supported
6305 MachineInstr
*llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair
&P
,
6306 MachineRegisterInfo
&MRI
) {
6307 assert(MRI
.isSSA());
6308 if (!Register::isVirtualRegister(P
.Reg
))
6312 auto *DefInst
= MRI
.getVRegDef(RSR
.Reg
);
6313 while (auto *MI
= DefInst
) {
6315 switch (MI
->getOpcode()) {
6317 case AMDGPU::V_MOV_B32_e32
: {
6318 auto &Op1
= MI
->getOperand(1);
6319 if (Op1
.isReg() && Register::isVirtualRegister(Op1
.getReg())) {
6322 RSR
= getRegSubRegPair(Op1
);
6323 DefInst
= MRI
.getVRegDef(RSR
.Reg
);
6328 if (followSubRegDef(*MI
, RSR
)) {
6331 DefInst
= MRI
.getVRegDef(RSR
.Reg
);
6340 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo
&MRI
,
6342 const MachineInstr
&DefMI
,
6343 const MachineInstr
&UseMI
) {
6344 assert(MRI
.isSSA() && "Must be run on SSA");
6346 auto *TRI
= MRI
.getTargetRegisterInfo();
6347 auto *DefBB
= DefMI
.getParent();
6349 // Don't bother searching between blocks, although it is possible this block
6350 // doesn't modify exec.
6351 if (UseMI
.getParent() != DefBB
)
6354 const int MaxInstScan
= 20;
6357 // Stop scan at the use.
6358 auto E
= UseMI
.getIterator();
6359 for (auto I
= std::next(DefMI
.getIterator()); I
!= E
; ++I
) {
6360 if (I
->isDebugInstr())
6363 if (++NumInst
> MaxInstScan
)
6366 if (I
->modifiesRegister(AMDGPU::EXEC
, TRI
))
6373 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo
&MRI
,
6375 const MachineInstr
&DefMI
) {
6376 assert(MRI
.isSSA() && "Must be run on SSA");
6378 auto *TRI
= MRI
.getTargetRegisterInfo();
6379 auto *DefBB
= DefMI
.getParent();
6381 const int MaxUseInstScan
= 10;
6384 for (auto &UseInst
: MRI
.use_nodbg_instructions(VReg
)) {
6385 // Don't bother searching between blocks, although it is possible this block
6386 // doesn't modify exec.
6387 if (UseInst
.getParent() != DefBB
)
6390 if (++NumUseInst
> MaxUseInstScan
)
6394 const int MaxInstScan
= 20;
6397 // Stop scan when we have seen all the uses.
6398 for (auto I
= std::next(DefMI
.getIterator()); ; ++I
) {
6399 if (I
->isDebugInstr())
6402 if (++NumInst
> MaxInstScan
)
6405 if (I
->readsRegister(VReg
))
6406 if (--NumUseInst
== 0)
6409 if (I
->modifiesRegister(AMDGPU::EXEC
, TRI
))
6414 MachineInstr
*SIInstrInfo::createPHIDestinationCopy(
6415 MachineBasicBlock
&MBB
, MachineBasicBlock::iterator LastPHIIt
,
6416 const DebugLoc
&DL
, Register Src
, Register Dst
) const {
6417 auto Cur
= MBB
.begin();
6418 if (Cur
!= MBB
.end())
6420 if (!Cur
->isPHI() && Cur
->readsRegister(Dst
))
6421 return BuildMI(MBB
, Cur
, DL
, get(TargetOpcode::COPY
), Dst
).addReg(Src
);
6423 } while (Cur
!= MBB
.end() && Cur
!= LastPHIIt
);
6425 return TargetInstrInfo::createPHIDestinationCopy(MBB
, LastPHIIt
, DL
, Src
,
6429 MachineInstr
*SIInstrInfo::createPHISourceCopy(
6430 MachineBasicBlock
&MBB
, MachineBasicBlock::iterator InsPt
,
6431 const DebugLoc
&DL
, Register Src
, Register SrcSubReg
, Register Dst
) const {
6432 if (InsPt
!= MBB
.end() &&
6433 (InsPt
->getOpcode() == AMDGPU::SI_IF
||
6434 InsPt
->getOpcode() == AMDGPU::SI_ELSE
||
6435 InsPt
->getOpcode() == AMDGPU::SI_IF_BREAK
) &&
6436 InsPt
->definesRegister(Src
)) {
6438 return BuildMI(MBB
, InsPt
, InsPt
->getDebugLoc(),
6439 get(ST
.isWave32() ? AMDGPU::S_MOV_B32_term
6440 : AMDGPU::S_MOV_B64_term
),
6442 .addReg(Src
, 0, SrcSubReg
)
6443 .addReg(AMDGPU::EXEC
, RegState::Implicit
);
6445 return TargetInstrInfo::createPHISourceCopy(MBB
, InsPt
, DL
, Src
, SrcSubReg
,
6449 bool llvm::SIInstrInfo::isWave32() const { return ST
.isWave32(); }