[obj2yaml] - Fix BB after r373315.
[llvm-complete.git] / lib / Target / AMDGPU / SIInstrInfo.cpp
blobb6a90241d4de95c4da40e693ca745ce9d287a87b
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
12 //===----------------------------------------------------------------------===//
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/ADT/iterator_range.h"
28 #include "llvm/Analysis/AliasAnalysis.h"
29 #include "llvm/Analysis/MemoryLocation.h"
30 #include "llvm/Analysis/ValueTracking.h"
31 #include "llvm/CodeGen/MachineBasicBlock.h"
32 #include "llvm/CodeGen/MachineDominators.h"
33 #include "llvm/CodeGen/MachineFrameInfo.h"
34 #include "llvm/CodeGen/MachineFunction.h"
35 #include "llvm/CodeGen/MachineInstr.h"
36 #include "llvm/CodeGen/MachineInstrBuilder.h"
37 #include "llvm/CodeGen/MachineInstrBundle.h"
38 #include "llvm/CodeGen/MachineMemOperand.h"
39 #include "llvm/CodeGen/MachineOperand.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/RegisterScavenging.h"
42 #include "llvm/CodeGen/ScheduleDAG.h"
43 #include "llvm/CodeGen/SelectionDAGNodes.h"
44 #include "llvm/CodeGen/TargetOpcodes.h"
45 #include "llvm/CodeGen/TargetRegisterInfo.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/CommandLine.h"
54 #include "llvm/Support/Compiler.h"
55 #include "llvm/Support/ErrorHandling.h"
56 #include "llvm/Support/MachineValueType.h"
57 #include "llvm/Support/MathExtras.h"
58 #include "llvm/Target/TargetMachine.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
64 using namespace llvm;
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84 cl::desc("Restrict range of branch instructions (DEBUG)"));
86 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
87 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88 RI(ST), ST(ST) {}
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95 unsigned N = Node->getNumOperands();
96 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97 --N;
98 return N;
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104 unsigned Opc0 = N0->getMachineOpcode();
105 unsigned Opc1 = N1->getMachineOpcode();
107 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
110 if (Op0Idx == -1 && Op1Idx == -1)
111 return true;
114 if ((Op0Idx == -1 && Op1Idx != -1) ||
115 (Op1Idx == -1 && Op0Idx != -1))
116 return false;
118 // getNamedOperandIdx returns the index for the MachineInstr's operands,
119 // which includes the result as the first operand. We are indexing into the
120 // MachineSDNode's operands, so we need to skip the result operand to get
121 // the real index.
122 --Op0Idx;
123 --Op1Idx;
125 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
128 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
129 AliasAnalysis *AA) const {
130 // TODO: The generic check fails for VALU instructions that should be
131 // rematerializable due to implicit reads of exec. We really want all of the
132 // generic logic for this except for this.
133 switch (MI.getOpcode()) {
134 case AMDGPU::V_MOV_B32_e32:
135 case AMDGPU::V_MOV_B32_e64:
136 case AMDGPU::V_MOV_B64_PSEUDO:
137 // No implicit operands.
138 return MI.getNumOperands() == MI.getDesc().getNumOperands();
139 default:
140 return false;
144 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
145 int64_t &Offset0,
146 int64_t &Offset1) const {
147 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148 return false;
150 unsigned Opc0 = Load0->getMachineOpcode();
151 unsigned Opc1 = Load1->getMachineOpcode();
153 // Make sure both are actually loads.
154 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155 return false;
157 if (isDS(Opc0) && isDS(Opc1)) {
159 // FIXME: Handle this case:
160 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161 return false;
163 // Check base reg.
164 if (Load0->getOperand(0) != Load1->getOperand(0))
165 return false;
167 // Skip read2 / write2 variants for simplicity.
168 // TODO: We should report true if the used offsets are adjacent (excluded
169 // st64 versions).
170 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172 if (Offset0Idx == -1 || Offset1Idx == -1)
173 return false;
175 // XXX - be careful of datalesss loads
176 // getNamedOperandIdx returns the index for MachineInstrs. Since they
177 // include the output in the operand list, but SDNodes don't, we need to
178 // subtract the index by one.
179 Offset0Idx -= get(Opc0).NumDefs;
180 Offset1Idx -= get(Opc1).NumDefs;
181 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183 return true;
186 if (isSMRD(Opc0) && isSMRD(Opc1)) {
187 // Skip time and cache invalidation instructions.
188 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190 return false;
192 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1));
194 // Check base reg.
195 if (Load0->getOperand(0) != Load1->getOperand(0))
196 return false;
198 const ConstantSDNode *Load0Offset =
199 dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200 const ConstantSDNode *Load1Offset =
201 dyn_cast<ConstantSDNode>(Load1->getOperand(1));
203 if (!Load0Offset || !Load1Offset)
204 return false;
206 Offset0 = Load0Offset->getZExtValue();
207 Offset1 = Load1Offset->getZExtValue();
208 return true;
211 // MUBUF and MTBUF can access the same addresses.
212 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
214 // MUBUF and MTBUF have vaddr at different indices.
215 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218 return false;
220 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
223 if (OffIdx0 == -1 || OffIdx1 == -1)
224 return false;
226 // getNamedOperandIdx returns the index for MachineInstrs. Since they
227 // include the output in the operand list, but SDNodes don't, we need to
228 // subtract the index by one.
229 OffIdx0 -= get(Opc0).NumDefs;
230 OffIdx1 -= get(Opc1).NumDefs;
232 SDValue Off0 = Load0->getOperand(OffIdx0);
233 SDValue Off1 = Load1->getOperand(OffIdx1);
235 // The offset might be a FrameIndexSDNode.
236 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237 return false;
239 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241 return true;
244 return false;
247 static bool isStride64(unsigned Opc) {
248 switch (Opc) {
249 case AMDGPU::DS_READ2ST64_B32:
250 case AMDGPU::DS_READ2ST64_B64:
251 case AMDGPU::DS_WRITE2ST64_B32:
252 case AMDGPU::DS_WRITE2ST64_B64:
253 return true;
254 default:
255 return false;
259 bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt,
260 const MachineOperand *&BaseOp,
261 int64_t &Offset,
262 const TargetRegisterInfo *TRI) const {
263 unsigned Opc = LdSt.getOpcode();
265 if (isDS(LdSt)) {
266 const MachineOperand *OffsetImm =
267 getNamedOperand(LdSt, AMDGPU::OpName::offset);
268 if (OffsetImm) {
269 // Normal, single offset LDS instruction.
270 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271 // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272 // report that here?
273 if (!BaseOp)
274 return false;
276 Offset = OffsetImm->getImm();
277 assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278 "operands of type register.");
279 return true;
282 // The 2 offset instructions use offset0 and offset1 instead. We can treat
283 // these as a load with a single offset if the 2 offsets are consecutive. We
284 // will use this for some partially aligned loads.
285 const MachineOperand *Offset0Imm =
286 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287 const MachineOperand *Offset1Imm =
288 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
290 uint8_t Offset0 = Offset0Imm->getImm();
291 uint8_t Offset1 = Offset1Imm->getImm();
293 if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294 // Each of these offsets is in element sized units, so we need to convert
295 // to bytes of the individual reads.
297 unsigned EltSize;
298 if (LdSt.mayLoad())
299 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300 else {
301 assert(LdSt.mayStore());
302 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
306 if (isStride64(Opc))
307 EltSize *= 64;
309 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310 Offset = EltSize * Offset0;
311 assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312 "operands of type register.");
313 return true;
316 return false;
319 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321 if (SOffset && SOffset->isReg()) {
322 // We can only handle this if it's a stack access, as any other resource
323 // would require reporting multiple base registers.
324 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325 if (AddrReg && !AddrReg->isFI())
326 return false;
328 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
329 const SIMachineFunctionInfo *MFI
330 = LdSt.getParent()->getParent()->getInfo<SIMachineFunctionInfo>();
331 if (RSrc->getReg() != MFI->getScratchRSrcReg())
332 return false;
334 const MachineOperand *OffsetImm =
335 getNamedOperand(LdSt, AMDGPU::OpName::offset);
336 BaseOp = SOffset;
337 Offset = OffsetImm->getImm();
338 return true;
341 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
342 if (!AddrReg)
343 return false;
345 const MachineOperand *OffsetImm =
346 getNamedOperand(LdSt, AMDGPU::OpName::offset);
347 BaseOp = AddrReg;
348 Offset = OffsetImm->getImm();
350 if (SOffset) // soffset can be an inline immediate.
351 Offset += SOffset->getImm();
353 assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
354 "operands of type register.");
355 return true;
358 if (isSMRD(LdSt)) {
359 const MachineOperand *OffsetImm =
360 getNamedOperand(LdSt, AMDGPU::OpName::offset);
361 if (!OffsetImm)
362 return false;
364 const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
365 BaseOp = SBaseReg;
366 Offset = OffsetImm->getImm();
367 assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
368 "operands of type register.");
369 return true;
372 if (isFLAT(LdSt)) {
373 const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
374 if (VAddr) {
375 // Can't analyze 2 offsets.
376 if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
377 return false;
379 BaseOp = VAddr;
380 } else {
381 // scratch instructions have either vaddr or saddr.
382 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
385 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
386 assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
387 "operands of type register.");
388 return true;
391 return false;
394 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
395 const MachineOperand &BaseOp1,
396 const MachineInstr &MI2,
397 const MachineOperand &BaseOp2) {
398 // Support only base operands with base registers.
399 // Note: this could be extended to support FI operands.
400 if (!BaseOp1.isReg() || !BaseOp2.isReg())
401 return false;
403 if (BaseOp1.isIdenticalTo(BaseOp2))
404 return true;
406 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
407 return false;
409 auto MO1 = *MI1.memoperands_begin();
410 auto MO2 = *MI2.memoperands_begin();
411 if (MO1->getAddrSpace() != MO2->getAddrSpace())
412 return false;
414 auto Base1 = MO1->getValue();
415 auto Base2 = MO2->getValue();
416 if (!Base1 || !Base2)
417 return false;
418 const MachineFunction &MF = *MI1.getParent()->getParent();
419 const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
420 Base1 = GetUnderlyingObject(Base1, DL);
421 Base2 = GetUnderlyingObject(Base1, DL);
423 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
424 return false;
426 return Base1 == Base2;
429 bool SIInstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1,
430 const MachineOperand &BaseOp2,
431 unsigned NumLoads) const {
432 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
433 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
435 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
436 return false;
438 const MachineOperand *FirstDst = nullptr;
439 const MachineOperand *SecondDst = nullptr;
441 if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
442 (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
443 (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
444 const unsigned MaxGlobalLoadCluster = 6;
445 if (NumLoads > MaxGlobalLoadCluster)
446 return false;
448 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
449 if (!FirstDst)
450 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
451 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
452 if (!SecondDst)
453 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
454 } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
455 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
456 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
457 } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
458 FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
459 SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
462 if (!FirstDst || !SecondDst)
463 return false;
465 // Try to limit clustering based on the total number of bytes loaded
466 // rather than the number of instructions. This is done to help reduce
467 // register pressure. The method used is somewhat inexact, though,
468 // because it assumes that all loads in the cluster will load the
469 // same number of bytes as FirstLdSt.
471 // The unit of this value is bytes.
472 // FIXME: This needs finer tuning.
473 unsigned LoadClusterThreshold = 16;
475 const MachineRegisterInfo &MRI =
476 FirstLdSt.getParent()->getParent()->getRegInfo();
478 const Register Reg = FirstDst->getReg();
480 const TargetRegisterClass *DstRC = Register::isVirtualRegister(Reg)
481 ? MRI.getRegClass(Reg)
482 : RI.getPhysRegClass(Reg);
484 return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
487 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
488 // the first 16 loads will be interleaved with the stores, and the next 16 will
489 // be clustered as expected. It should really split into 2 16 store batches.
491 // Loads are clustered until this returns false, rather than trying to schedule
492 // groups of stores. This also means we have to deal with saying different
493 // address space loads should be clustered, and ones which might cause bank
494 // conflicts.
496 // This might be deprecated so it might not be worth that much effort to fix.
497 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
498 int64_t Offset0, int64_t Offset1,
499 unsigned NumLoads) const {
500 assert(Offset1 > Offset0 &&
501 "Second offset should be larger than first offset!");
502 // If we have less than 16 loads in a row, and the offsets are within 64
503 // bytes, then schedule together.
505 // A cacheline is 64 bytes (for global memory).
506 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
509 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
510 MachineBasicBlock::iterator MI,
511 const DebugLoc &DL, unsigned DestReg,
512 unsigned SrcReg, bool KillSrc) {
513 MachineFunction *MF = MBB.getParent();
514 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
515 "illegal SGPR to VGPR copy",
516 DL, DS_Error);
517 LLVMContext &C = MF->getFunction().getContext();
518 C.diagnose(IllegalCopy);
520 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
521 .addReg(SrcReg, getKillRegState(KillSrc));
524 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
525 MachineBasicBlock::iterator MI,
526 const DebugLoc &DL, unsigned DestReg,
527 unsigned SrcReg, bool KillSrc) const {
528 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
530 if (RC == &AMDGPU::VGPR_32RegClass) {
531 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
532 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
533 AMDGPU::AGPR_32RegClass.contains(SrcReg));
534 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
535 AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
536 BuildMI(MBB, MI, DL, get(Opc), DestReg)
537 .addReg(SrcReg, getKillRegState(KillSrc));
538 return;
541 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
542 RC == &AMDGPU::SReg_32RegClass) {
543 if (SrcReg == AMDGPU::SCC) {
544 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
545 .addImm(-1)
546 .addImm(0);
547 return;
550 if (DestReg == AMDGPU::VCC_LO) {
551 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
552 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
553 .addReg(SrcReg, getKillRegState(KillSrc));
554 } else {
555 // FIXME: Hack until VReg_1 removed.
556 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
557 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
558 .addImm(0)
559 .addReg(SrcReg, getKillRegState(KillSrc));
562 return;
565 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
566 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
567 return;
570 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
571 .addReg(SrcReg, getKillRegState(KillSrc));
572 return;
575 if (RC == &AMDGPU::SReg_64RegClass) {
576 if (DestReg == AMDGPU::VCC) {
577 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
578 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
579 .addReg(SrcReg, getKillRegState(KillSrc));
580 } else {
581 // FIXME: Hack until VReg_1 removed.
582 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
583 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
584 .addImm(0)
585 .addReg(SrcReg, getKillRegState(KillSrc));
588 return;
591 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
592 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
593 return;
596 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
597 .addReg(SrcReg, getKillRegState(KillSrc));
598 return;
601 if (DestReg == AMDGPU::SCC) {
602 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
603 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
604 .addReg(SrcReg, getKillRegState(KillSrc))
605 .addImm(0);
606 return;
609 if (RC == &AMDGPU::AGPR_32RegClass) {
610 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
611 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
612 AMDGPU::AGPR_32RegClass.contains(SrcReg));
613 if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
614 // First try to find defining accvgpr_write to avoid temporary registers.
615 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
616 --Def;
617 if (!Def->definesRegister(SrcReg, &RI))
618 continue;
619 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
620 break;
622 MachineOperand &DefOp = Def->getOperand(1);
623 assert(DefOp.isReg() || DefOp.isImm());
625 if (DefOp.isReg()) {
626 // Check that register source operand if not clobbered before MI.
627 // Immediate operands are always safe to propagate.
628 bool SafeToPropagate = true;
629 for (auto I = Def; I != MI && SafeToPropagate; ++I)
630 if (I->modifiesRegister(DefOp.getReg(), &RI))
631 SafeToPropagate = false;
633 if (!SafeToPropagate)
634 break;
636 DefOp.setIsKill(false);
639 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
640 .add(DefOp);
641 return;
644 RegScavenger RS;
645 RS.enterBasicBlock(MBB);
646 RS.forward(MI);
648 // Ideally we want to have three registers for a long reg_sequence copy
649 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
650 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
651 *MBB.getParent());
653 // Registers in the sequence are allocated contiguously so we can just
654 // use register number to pick one of three round-robin temps.
655 unsigned RegNo = DestReg % 3;
656 unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
657 if (!Tmp)
658 report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
659 RS.setRegUsed(Tmp);
660 // Only loop through if there are any free registers left, otherwise
661 // scavenger may report a fatal error without emergency spill slot
662 // or spill with the slot.
663 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
664 unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
665 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
666 break;
667 Tmp = Tmp2;
668 RS.setRegUsed(Tmp);
670 copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
671 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
672 .addReg(Tmp, RegState::Kill);
673 return;
676 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
677 .addReg(SrcReg, getKillRegState(KillSrc));
678 return;
681 unsigned EltSize = 4;
682 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
683 if (RI.isSGPRClass(RC)) {
684 // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
685 if (!(RI.getRegSizeInBits(*RC) % 64)) {
686 Opcode = AMDGPU::S_MOV_B64;
687 EltSize = 8;
688 } else {
689 Opcode = AMDGPU::S_MOV_B32;
690 EltSize = 4;
693 if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
694 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
695 return;
697 } else if (RI.hasAGPRs(RC)) {
698 Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
699 AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
700 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
701 Opcode = AMDGPU::V_ACCVGPR_READ_B32;
704 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
705 bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
707 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
708 unsigned SubIdx;
709 if (Forward)
710 SubIdx = SubIndices[Idx];
711 else
712 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
714 if (Opcode == TargetOpcode::COPY) {
715 copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
716 RI.getSubReg(SrcReg, SubIdx), KillSrc);
717 continue;
720 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
721 get(Opcode), RI.getSubReg(DestReg, SubIdx));
723 Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
725 if (Idx == 0)
726 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
728 bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
729 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
733 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
734 int NewOpc;
736 // Try to map original to commuted opcode
737 NewOpc = AMDGPU::getCommuteRev(Opcode);
738 if (NewOpc != -1)
739 // Check if the commuted (REV) opcode exists on the target.
740 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
742 // Try to map commuted to original opcode
743 NewOpc = AMDGPU::getCommuteOrig(Opcode);
744 if (NewOpc != -1)
745 // Check if the original (non-REV) opcode exists on the target.
746 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
748 return Opcode;
751 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
752 MachineBasicBlock::iterator MI,
753 const DebugLoc &DL, unsigned DestReg,
754 int64_t Value) const {
755 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
756 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
757 if (RegClass == &AMDGPU::SReg_32RegClass ||
758 RegClass == &AMDGPU::SGPR_32RegClass ||
759 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
760 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
761 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
762 .addImm(Value);
763 return;
766 if (RegClass == &AMDGPU::SReg_64RegClass ||
767 RegClass == &AMDGPU::SGPR_64RegClass ||
768 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
769 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
770 .addImm(Value);
771 return;
774 if (RegClass == &AMDGPU::VGPR_32RegClass) {
775 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
776 .addImm(Value);
777 return;
779 if (RegClass == &AMDGPU::VReg_64RegClass) {
780 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
781 .addImm(Value);
782 return;
785 unsigned EltSize = 4;
786 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
787 if (RI.isSGPRClass(RegClass)) {
788 if (RI.getRegSizeInBits(*RegClass) > 32) {
789 Opcode = AMDGPU::S_MOV_B64;
790 EltSize = 8;
791 } else {
792 Opcode = AMDGPU::S_MOV_B32;
793 EltSize = 4;
797 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
798 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
799 int64_t IdxValue = Idx == 0 ? Value : 0;
801 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
802 get(Opcode), RI.getSubReg(DestReg, Idx));
803 Builder.addImm(IdxValue);
807 const TargetRegisterClass *
808 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
809 return &AMDGPU::VGPR_32RegClass;
812 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
813 MachineBasicBlock::iterator I,
814 const DebugLoc &DL, unsigned DstReg,
815 ArrayRef<MachineOperand> Cond,
816 unsigned TrueReg,
817 unsigned FalseReg) const {
818 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
819 MachineFunction *MF = MBB.getParent();
820 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
821 const TargetRegisterClass *BoolXExecRC =
822 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
823 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
824 "Not a VGPR32 reg");
826 if (Cond.size() == 1) {
827 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
828 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
829 .add(Cond[0]);
830 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
831 .addImm(0)
832 .addReg(FalseReg)
833 .addImm(0)
834 .addReg(TrueReg)
835 .addReg(SReg);
836 } else if (Cond.size() == 2) {
837 assert(Cond[0].isImm() && "Cond[0] is not an immediate");
838 switch (Cond[0].getImm()) {
839 case SIInstrInfo::SCC_TRUE: {
840 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
841 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
842 : AMDGPU::S_CSELECT_B64), SReg)
843 .addImm(-1)
844 .addImm(0);
845 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
846 .addImm(0)
847 .addReg(FalseReg)
848 .addImm(0)
849 .addReg(TrueReg)
850 .addReg(SReg);
851 break;
853 case SIInstrInfo::SCC_FALSE: {
854 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
855 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
856 : AMDGPU::S_CSELECT_B64), SReg)
857 .addImm(0)
858 .addImm(-1);
859 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
860 .addImm(0)
861 .addReg(FalseReg)
862 .addImm(0)
863 .addReg(TrueReg)
864 .addReg(SReg);
865 break;
867 case SIInstrInfo::VCCNZ: {
868 MachineOperand RegOp = Cond[1];
869 RegOp.setImplicit(false);
870 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
871 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
872 .add(RegOp);
873 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
874 .addImm(0)
875 .addReg(FalseReg)
876 .addImm(0)
877 .addReg(TrueReg)
878 .addReg(SReg);
879 break;
881 case SIInstrInfo::VCCZ: {
882 MachineOperand RegOp = Cond[1];
883 RegOp.setImplicit(false);
884 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
885 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
886 .add(RegOp);
887 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
888 .addImm(0)
889 .addReg(TrueReg)
890 .addImm(0)
891 .addReg(FalseReg)
892 .addReg(SReg);
893 break;
895 case SIInstrInfo::EXECNZ: {
896 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
897 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
898 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
899 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
900 .addImm(0);
901 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
902 : AMDGPU::S_CSELECT_B64), SReg)
903 .addImm(-1)
904 .addImm(0);
905 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
906 .addImm(0)
907 .addReg(FalseReg)
908 .addImm(0)
909 .addReg(TrueReg)
910 .addReg(SReg);
911 break;
913 case SIInstrInfo::EXECZ: {
914 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
915 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
916 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
917 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
918 .addImm(0);
919 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
920 : AMDGPU::S_CSELECT_B64), SReg)
921 .addImm(0)
922 .addImm(-1);
923 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
924 .addImm(0)
925 .addReg(FalseReg)
926 .addImm(0)
927 .addReg(TrueReg)
928 .addReg(SReg);
929 llvm_unreachable("Unhandled branch predicate EXECZ");
930 break;
932 default:
933 llvm_unreachable("invalid branch predicate");
935 } else {
936 llvm_unreachable("Can only handle Cond size 1 or 2");
940 unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
941 MachineBasicBlock::iterator I,
942 const DebugLoc &DL,
943 unsigned SrcReg, int Value) const {
944 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
945 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
946 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
947 .addImm(Value)
948 .addReg(SrcReg);
950 return Reg;
953 unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
954 MachineBasicBlock::iterator I,
955 const DebugLoc &DL,
956 unsigned SrcReg, int Value) const {
957 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
958 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
959 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
960 .addImm(Value)
961 .addReg(SrcReg);
963 return Reg;
966 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
968 if (RI.hasAGPRs(DstRC))
969 return AMDGPU::COPY;
970 if (RI.getRegSizeInBits(*DstRC) == 32) {
971 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
972 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
973 return AMDGPU::S_MOV_B64;
974 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
975 return AMDGPU::V_MOV_B64_PSEUDO;
977 return AMDGPU::COPY;
980 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
981 switch (Size) {
982 case 4:
983 return AMDGPU::SI_SPILL_S32_SAVE;
984 case 8:
985 return AMDGPU::SI_SPILL_S64_SAVE;
986 case 12:
987 return AMDGPU::SI_SPILL_S96_SAVE;
988 case 16:
989 return AMDGPU::SI_SPILL_S128_SAVE;
990 case 20:
991 return AMDGPU::SI_SPILL_S160_SAVE;
992 case 32:
993 return AMDGPU::SI_SPILL_S256_SAVE;
994 case 64:
995 return AMDGPU::SI_SPILL_S512_SAVE;
996 case 128:
997 return AMDGPU::SI_SPILL_S1024_SAVE;
998 default:
999 llvm_unreachable("unknown register size");
1003 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1004 switch (Size) {
1005 case 4:
1006 return AMDGPU::SI_SPILL_V32_SAVE;
1007 case 8:
1008 return AMDGPU::SI_SPILL_V64_SAVE;
1009 case 12:
1010 return AMDGPU::SI_SPILL_V96_SAVE;
1011 case 16:
1012 return AMDGPU::SI_SPILL_V128_SAVE;
1013 case 20:
1014 return AMDGPU::SI_SPILL_V160_SAVE;
1015 case 32:
1016 return AMDGPU::SI_SPILL_V256_SAVE;
1017 case 64:
1018 return AMDGPU::SI_SPILL_V512_SAVE;
1019 case 128:
1020 return AMDGPU::SI_SPILL_V1024_SAVE;
1021 default:
1022 llvm_unreachable("unknown register size");
1026 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1027 switch (Size) {
1028 case 4:
1029 return AMDGPU::SI_SPILL_A32_SAVE;
1030 case 8:
1031 return AMDGPU::SI_SPILL_A64_SAVE;
1032 case 16:
1033 return AMDGPU::SI_SPILL_A128_SAVE;
1034 case 64:
1035 return AMDGPU::SI_SPILL_A512_SAVE;
1036 case 128:
1037 return AMDGPU::SI_SPILL_A1024_SAVE;
1038 default:
1039 llvm_unreachable("unknown register size");
1043 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1044 MachineBasicBlock::iterator MI,
1045 unsigned SrcReg, bool isKill,
1046 int FrameIndex,
1047 const TargetRegisterClass *RC,
1048 const TargetRegisterInfo *TRI) const {
1049 MachineFunction *MF = MBB.getParent();
1050 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1051 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1052 const DebugLoc &DL = MBB.findDebugLoc(MI);
1054 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1055 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1056 MachinePointerInfo PtrInfo
1057 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1058 MachineMemOperand *MMO
1059 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
1060 Size, Align);
1061 unsigned SpillSize = TRI->getSpillSize(*RC);
1063 if (RI.isSGPRClass(RC)) {
1064 MFI->setHasSpilledSGPRs();
1066 // We are only allowed to create one new instruction when spilling
1067 // registers, so we need to use pseudo instruction for spilling SGPRs.
1068 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1070 // The SGPR spill/restore instructions only work on number sgprs, so we need
1071 // to make sure we are using the correct register class.
1072 if (Register::isVirtualRegister(SrcReg) && SpillSize == 4) {
1073 MachineRegisterInfo &MRI = MF->getRegInfo();
1074 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1077 MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
1078 .addReg(SrcReg, getKillRegState(isKill)) // data
1079 .addFrameIndex(FrameIndex) // addr
1080 .addMemOperand(MMO)
1081 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1082 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1083 // Add the scratch resource registers as implicit uses because we may end up
1084 // needing them, and need to ensure that the reserved registers are
1085 // correctly handled.
1086 if (RI.spillSGPRToVGPR())
1087 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1088 if (ST.hasScalarStores()) {
1089 // m0 is used for offset to scalar stores if used to spill.
1090 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1093 return;
1096 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1097 : getVGPRSpillSaveOpcode(SpillSize);
1098 MFI->setHasSpilledVGPRs();
1100 auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1101 if (RI.hasAGPRs(RC)) {
1102 MachineRegisterInfo &MRI = MF->getRegInfo();
1103 Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1104 MIB.addReg(Tmp, RegState::Define);
1106 MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1107 .addFrameIndex(FrameIndex) // addr
1108 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1109 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1110 .addImm(0) // offset
1111 .addMemOperand(MMO);
1114 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1115 switch (Size) {
1116 case 4:
1117 return AMDGPU::SI_SPILL_S32_RESTORE;
1118 case 8:
1119 return AMDGPU::SI_SPILL_S64_RESTORE;
1120 case 12:
1121 return AMDGPU::SI_SPILL_S96_RESTORE;
1122 case 16:
1123 return AMDGPU::SI_SPILL_S128_RESTORE;
1124 case 20:
1125 return AMDGPU::SI_SPILL_S160_RESTORE;
1126 case 32:
1127 return AMDGPU::SI_SPILL_S256_RESTORE;
1128 case 64:
1129 return AMDGPU::SI_SPILL_S512_RESTORE;
1130 case 128:
1131 return AMDGPU::SI_SPILL_S1024_RESTORE;
1132 default:
1133 llvm_unreachable("unknown register size");
1137 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1138 switch (Size) {
1139 case 4:
1140 return AMDGPU::SI_SPILL_V32_RESTORE;
1141 case 8:
1142 return AMDGPU::SI_SPILL_V64_RESTORE;
1143 case 12:
1144 return AMDGPU::SI_SPILL_V96_RESTORE;
1145 case 16:
1146 return AMDGPU::SI_SPILL_V128_RESTORE;
1147 case 20:
1148 return AMDGPU::SI_SPILL_V160_RESTORE;
1149 case 32:
1150 return AMDGPU::SI_SPILL_V256_RESTORE;
1151 case 64:
1152 return AMDGPU::SI_SPILL_V512_RESTORE;
1153 case 128:
1154 return AMDGPU::SI_SPILL_V1024_RESTORE;
1155 default:
1156 llvm_unreachable("unknown register size");
1160 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1161 switch (Size) {
1162 case 4:
1163 return AMDGPU::SI_SPILL_A32_RESTORE;
1164 case 8:
1165 return AMDGPU::SI_SPILL_A64_RESTORE;
1166 case 16:
1167 return AMDGPU::SI_SPILL_A128_RESTORE;
1168 case 64:
1169 return AMDGPU::SI_SPILL_A512_RESTORE;
1170 case 128:
1171 return AMDGPU::SI_SPILL_A1024_RESTORE;
1172 default:
1173 llvm_unreachable("unknown register size");
1177 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1178 MachineBasicBlock::iterator MI,
1179 unsigned DestReg, int FrameIndex,
1180 const TargetRegisterClass *RC,
1181 const TargetRegisterInfo *TRI) const {
1182 MachineFunction *MF = MBB.getParent();
1183 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1184 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1185 const DebugLoc &DL = MBB.findDebugLoc(MI);
1186 unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1187 unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1188 unsigned SpillSize = TRI->getSpillSize(*RC);
1190 MachinePointerInfo PtrInfo
1191 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1193 MachineMemOperand *MMO = MF->getMachineMemOperand(
1194 PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1196 if (RI.isSGPRClass(RC)) {
1197 MFI->setHasSpilledSGPRs();
1199 // FIXME: Maybe this should not include a memoperand because it will be
1200 // lowered to non-memory instructions.
1201 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1202 if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
1203 MachineRegisterInfo &MRI = MF->getRegInfo();
1204 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1207 if (RI.spillSGPRToVGPR())
1208 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1209 MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1210 .addFrameIndex(FrameIndex) // addr
1211 .addMemOperand(MMO)
1212 .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1213 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1215 if (ST.hasScalarStores()) {
1216 // m0 is used for offset to scalar stores if used to spill.
1217 Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1220 return;
1223 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1224 : getVGPRSpillRestoreOpcode(SpillSize);
1225 auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1226 if (RI.hasAGPRs(RC)) {
1227 MachineRegisterInfo &MRI = MF->getRegInfo();
1228 Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1229 MIB.addReg(Tmp, RegState::Define);
1231 MIB.addFrameIndex(FrameIndex) // vaddr
1232 .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1233 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1234 .addImm(0) // offset
1235 .addMemOperand(MMO);
1238 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1239 unsigned SIInstrInfo::calculateLDSSpillAddress(
1240 MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1241 unsigned FrameOffset, unsigned Size) const {
1242 MachineFunction *MF = MBB.getParent();
1243 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1244 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1245 const DebugLoc &DL = MBB.findDebugLoc(MI);
1246 unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1247 unsigned WavefrontSize = ST.getWavefrontSize();
1249 unsigned TIDReg = MFI->getTIDReg();
1250 if (!MFI->hasCalculatedTID()) {
1251 MachineBasicBlock &Entry = MBB.getParent()->front();
1252 MachineBasicBlock::iterator Insert = Entry.front();
1253 const DebugLoc &DL = Insert->getDebugLoc();
1255 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1256 *MF);
1257 if (TIDReg == AMDGPU::NoRegister)
1258 return TIDReg;
1260 if (!AMDGPU::isShader(MF->getFunction().getCallingConv()) &&
1261 WorkGroupSize > WavefrontSize) {
1262 Register TIDIGXReg =
1263 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1264 Register TIDIGYReg =
1265 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1266 Register TIDIGZReg =
1267 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
1268 Register InputPtrReg =
1269 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1270 for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1271 if (!Entry.isLiveIn(Reg))
1272 Entry.addLiveIn(Reg);
1275 RS->enterBasicBlock(Entry);
1276 // FIXME: Can we scavenge an SReg_64 and access the subregs?
1277 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1278 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1279 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1280 .addReg(InputPtrReg)
1281 .addImm(SI::KernelInputOffsets::NGROUPS_Z);
1282 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1283 .addReg(InputPtrReg)
1284 .addImm(SI::KernelInputOffsets::NGROUPS_Y);
1286 // NGROUPS.X * NGROUPS.Y
1287 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1288 .addReg(STmp1)
1289 .addReg(STmp0);
1290 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1291 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1292 .addReg(STmp1)
1293 .addReg(TIDIGXReg);
1294 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1295 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1296 .addReg(STmp0)
1297 .addReg(TIDIGYReg)
1298 .addReg(TIDReg);
1299 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1300 getAddNoCarry(Entry, Insert, DL, TIDReg)
1301 .addReg(TIDReg)
1302 .addReg(TIDIGZReg)
1303 .addImm(0); // clamp bit
1304 } else {
1305 // Get the wave id
1306 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1307 TIDReg)
1308 .addImm(-1)
1309 .addImm(0);
1311 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1312 TIDReg)
1313 .addImm(-1)
1314 .addReg(TIDReg);
1317 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1318 TIDReg)
1319 .addImm(2)
1320 .addReg(TIDReg);
1321 MFI->setTIDReg(TIDReg);
1324 // Add FrameIndex to LDS offset
1325 unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1326 getAddNoCarry(MBB, MI, DL, TmpReg)
1327 .addImm(LDSOffset)
1328 .addReg(TIDReg)
1329 .addImm(0); // clamp bit
1331 return TmpReg;
1334 void SIInstrInfo::insertWaitStates(MachineBasicBlock &MBB,
1335 MachineBasicBlock::iterator MI,
1336 int Count) const {
1337 DebugLoc DL = MBB.findDebugLoc(MI);
1338 while (Count > 0) {
1339 int Arg;
1340 if (Count >= 8)
1341 Arg = 7;
1342 else
1343 Arg = Count - 1;
1344 Count -= 8;
1345 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1346 .addImm(Arg);
1350 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1351 MachineBasicBlock::iterator MI) const {
1352 insertWaitStates(MBB, MI, 1);
1355 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1356 auto MF = MBB.getParent();
1357 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1359 assert(Info->isEntryFunction());
1361 if (MBB.succ_empty()) {
1362 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1363 if (HasNoTerminator) {
1364 if (Info->returnsVoid()) {
1365 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1366 } else {
1367 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1373 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
1374 switch (MI.getOpcode()) {
1375 default: return 1; // FIXME: Do wait states equal cycles?
1377 case AMDGPU::S_NOP:
1378 return MI.getOperand(0).getImm() + 1;
1382 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1383 MachineBasicBlock &MBB = *MI.getParent();
1384 DebugLoc DL = MBB.findDebugLoc(MI);
1385 switch (MI.getOpcode()) {
1386 default: return TargetInstrInfo::expandPostRAPseudo(MI);
1387 case AMDGPU::S_MOV_B64_term:
1388 // This is only a terminator to get the correct spill code placement during
1389 // register allocation.
1390 MI.setDesc(get(AMDGPU::S_MOV_B64));
1391 break;
1393 case AMDGPU::S_MOV_B32_term:
1394 // This is only a terminator to get the correct spill code placement during
1395 // register allocation.
1396 MI.setDesc(get(AMDGPU::S_MOV_B32));
1397 break;
1399 case AMDGPU::S_XOR_B64_term:
1400 // This is only a terminator to get the correct spill code placement during
1401 // register allocation.
1402 MI.setDesc(get(AMDGPU::S_XOR_B64));
1403 break;
1405 case AMDGPU::S_XOR_B32_term:
1406 // This is only a terminator to get the correct spill code placement during
1407 // register allocation.
1408 MI.setDesc(get(AMDGPU::S_XOR_B32));
1409 break;
1411 case AMDGPU::S_OR_B32_term:
1412 // This is only a terminator to get the correct spill code placement during
1413 // register allocation.
1414 MI.setDesc(get(AMDGPU::S_OR_B32));
1415 break;
1417 case AMDGPU::S_ANDN2_B64_term:
1418 // This is only a terminator to get the correct spill code placement during
1419 // register allocation.
1420 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1421 break;
1423 case AMDGPU::S_ANDN2_B32_term:
1424 // This is only a terminator to get the correct spill code placement during
1425 // register allocation.
1426 MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1427 break;
1429 case AMDGPU::V_MOV_B64_PSEUDO: {
1430 Register Dst = MI.getOperand(0).getReg();
1431 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1432 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1434 const MachineOperand &SrcOp = MI.getOperand(1);
1435 // FIXME: Will this work for 64-bit floating point immediates?
1436 assert(!SrcOp.isFPImm());
1437 if (SrcOp.isImm()) {
1438 APInt Imm(64, SrcOp.getImm());
1439 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1440 .addImm(Imm.getLoBits(32).getZExtValue())
1441 .addReg(Dst, RegState::Implicit | RegState::Define);
1442 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1443 .addImm(Imm.getHiBits(32).getZExtValue())
1444 .addReg(Dst, RegState::Implicit | RegState::Define);
1445 } else {
1446 assert(SrcOp.isReg());
1447 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1448 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1449 .addReg(Dst, RegState::Implicit | RegState::Define);
1450 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1451 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1452 .addReg(Dst, RegState::Implicit | RegState::Define);
1454 MI.eraseFromParent();
1455 break;
1457 case AMDGPU::V_SET_INACTIVE_B32: {
1458 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1459 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1460 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1461 .addReg(Exec);
1462 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1463 .add(MI.getOperand(2));
1464 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1465 .addReg(Exec);
1466 MI.eraseFromParent();
1467 break;
1469 case AMDGPU::V_SET_INACTIVE_B64: {
1470 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1471 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1472 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1473 .addReg(Exec);
1474 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1475 MI.getOperand(0).getReg())
1476 .add(MI.getOperand(2));
1477 expandPostRAPseudo(*Copy);
1478 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1479 .addReg(Exec);
1480 MI.eraseFromParent();
1481 break;
1483 case AMDGPU::V_MOVRELD_B32_V1:
1484 case AMDGPU::V_MOVRELD_B32_V2:
1485 case AMDGPU::V_MOVRELD_B32_V4:
1486 case AMDGPU::V_MOVRELD_B32_V8:
1487 case AMDGPU::V_MOVRELD_B32_V16: {
1488 const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1489 Register VecReg = MI.getOperand(0).getReg();
1490 bool IsUndef = MI.getOperand(1).isUndef();
1491 unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1492 assert(VecReg == MI.getOperand(1).getReg());
1494 MachineInstr *MovRel =
1495 BuildMI(MBB, MI, DL, MovRelDesc)
1496 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1497 .add(MI.getOperand(2))
1498 .addReg(VecReg, RegState::ImplicitDefine)
1499 .addReg(VecReg,
1500 RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1502 const int ImpDefIdx =
1503 MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1504 const int ImpUseIdx = ImpDefIdx + 1;
1505 MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1507 MI.eraseFromParent();
1508 break;
1510 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1511 MachineFunction &MF = *MBB.getParent();
1512 Register Reg = MI.getOperand(0).getReg();
1513 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1514 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1516 // Create a bundle so these instructions won't be re-ordered by the
1517 // post-RA scheduler.
1518 MIBundleBuilder Bundler(MBB, MI);
1519 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1521 // Add 32-bit offset from this instruction to the start of the
1522 // constant data.
1523 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1524 .addReg(RegLo)
1525 .add(MI.getOperand(1)));
1527 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1528 .addReg(RegHi);
1529 MIB.add(MI.getOperand(2));
1531 Bundler.append(MIB);
1532 finalizeBundle(MBB, Bundler.begin());
1534 MI.eraseFromParent();
1535 break;
1537 case AMDGPU::ENTER_WWM: {
1538 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1539 // WWM is entered.
1540 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1541 : AMDGPU::S_OR_SAVEEXEC_B64));
1542 break;
1544 case AMDGPU::EXIT_WWM: {
1545 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1546 // WWM is exited.
1547 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1548 break;
1550 case TargetOpcode::BUNDLE: {
1551 if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1552 return false;
1554 // If it is a load it must be a memory clause
1555 for (MachineBasicBlock::instr_iterator I = MI.getIterator();
1556 I->isBundledWithSucc(); ++I) {
1557 I->unbundleFromSucc();
1558 for (MachineOperand &MO : I->operands())
1559 if (MO.isReg())
1560 MO.setIsInternalRead(false);
1563 MI.eraseFromParent();
1564 break;
1567 return true;
1570 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
1571 MachineOperand &Src0,
1572 unsigned Src0OpName,
1573 MachineOperand &Src1,
1574 unsigned Src1OpName) const {
1575 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1576 if (!Src0Mods)
1577 return false;
1579 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1580 assert(Src1Mods &&
1581 "All commutable instructions have both src0 and src1 modifiers");
1583 int Src0ModsVal = Src0Mods->getImm();
1584 int Src1ModsVal = Src1Mods->getImm();
1586 Src1Mods->setImm(Src0ModsVal);
1587 Src0Mods->setImm(Src1ModsVal);
1588 return true;
1591 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
1592 MachineOperand &RegOp,
1593 MachineOperand &NonRegOp) {
1594 Register Reg = RegOp.getReg();
1595 unsigned SubReg = RegOp.getSubReg();
1596 bool IsKill = RegOp.isKill();
1597 bool IsDead = RegOp.isDead();
1598 bool IsUndef = RegOp.isUndef();
1599 bool IsDebug = RegOp.isDebug();
1601 if (NonRegOp.isImm())
1602 RegOp.ChangeToImmediate(NonRegOp.getImm());
1603 else if (NonRegOp.isFI())
1604 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1605 else
1606 return nullptr;
1608 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1609 NonRegOp.setSubReg(SubReg);
1611 return &MI;
1614 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1615 unsigned Src0Idx,
1616 unsigned Src1Idx) const {
1617 assert(!NewMI && "this should never be used");
1619 unsigned Opc = MI.getOpcode();
1620 int CommutedOpcode = commuteOpcode(Opc);
1621 if (CommutedOpcode == -1)
1622 return nullptr;
1624 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1625 static_cast<int>(Src0Idx) &&
1626 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1627 static_cast<int>(Src1Idx) &&
1628 "inconsistency with findCommutedOpIndices");
1630 MachineOperand &Src0 = MI.getOperand(Src0Idx);
1631 MachineOperand &Src1 = MI.getOperand(Src1Idx);
1633 MachineInstr *CommutedMI = nullptr;
1634 if (Src0.isReg() && Src1.isReg()) {
1635 if (isOperandLegal(MI, Src1Idx, &Src0)) {
1636 // Be sure to copy the source modifiers to the right place.
1637 CommutedMI
1638 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1641 } else if (Src0.isReg() && !Src1.isReg()) {
1642 // src0 should always be able to support any operand type, so no need to
1643 // check operand legality.
1644 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1645 } else if (!Src0.isReg() && Src1.isReg()) {
1646 if (isOperandLegal(MI, Src1Idx, &Src0))
1647 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1648 } else {
1649 // FIXME: Found two non registers to commute. This does happen.
1650 return nullptr;
1653 if (CommutedMI) {
1654 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1655 Src1, AMDGPU::OpName::src1_modifiers);
1657 CommutedMI->setDesc(get(CommutedOpcode));
1660 return CommutedMI;
1663 // This needs to be implemented because the source modifiers may be inserted
1664 // between the true commutable operands, and the base
1665 // TargetInstrInfo::commuteInstruction uses it.
1666 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1667 unsigned &SrcOpIdx0,
1668 unsigned &SrcOpIdx1) const {
1669 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1672 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1673 unsigned &SrcOpIdx1) const {
1674 if (!Desc.isCommutable())
1675 return false;
1677 unsigned Opc = Desc.getOpcode();
1678 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1679 if (Src0Idx == -1)
1680 return false;
1682 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1683 if (Src1Idx == -1)
1684 return false;
1686 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1689 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1690 int64_t BrOffset) const {
1691 // BranchRelaxation should never have to check s_setpc_b64 because its dest
1692 // block is unanalyzable.
1693 assert(BranchOp != AMDGPU::S_SETPC_B64);
1695 // Convert to dwords.
1696 BrOffset /= 4;
1698 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1699 // from the next instruction.
1700 BrOffset -= 1;
1702 return isIntN(BranchOffsetBits, BrOffset);
1705 MachineBasicBlock *SIInstrInfo::getBranchDestBlock(
1706 const MachineInstr &MI) const {
1707 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1708 // This would be a difficult analysis to perform, but can always be legal so
1709 // there's no need to analyze it.
1710 return nullptr;
1713 return MI.getOperand(0).getMBB();
1716 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1717 MachineBasicBlock &DestBB,
1718 const DebugLoc &DL,
1719 int64_t BrOffset,
1720 RegScavenger *RS) const {
1721 assert(RS && "RegScavenger required for long branching");
1722 assert(MBB.empty() &&
1723 "new block should be inserted for expanding unconditional branch");
1724 assert(MBB.pred_size() == 1);
1726 MachineFunction *MF = MBB.getParent();
1727 MachineRegisterInfo &MRI = MF->getRegInfo();
1729 // FIXME: Virtual register workaround for RegScavenger not working with empty
1730 // blocks.
1731 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1733 auto I = MBB.end();
1735 // We need to compute the offset relative to the instruction immediately after
1736 // s_getpc_b64. Insert pc arithmetic code before last terminator.
1737 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1739 // TODO: Handle > 32-bit block address.
1740 if (BrOffset >= 0) {
1741 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1742 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1743 .addReg(PCReg, 0, AMDGPU::sub0)
1744 .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1745 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1746 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1747 .addReg(PCReg, 0, AMDGPU::sub1)
1748 .addImm(0);
1749 } else {
1750 // Backwards branch.
1751 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1752 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1753 .addReg(PCReg, 0, AMDGPU::sub0)
1754 .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1755 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1756 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1757 .addReg(PCReg, 0, AMDGPU::sub1)
1758 .addImm(0);
1761 // Insert the indirect branch after the other terminator.
1762 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1763 .addReg(PCReg);
1765 // FIXME: If spilling is necessary, this will fail because this scavenger has
1766 // no emergency stack slots. It is non-trivial to spill in this situation,
1767 // because the restore code needs to be specially placed after the
1768 // jump. BranchRelaxation then needs to be made aware of the newly inserted
1769 // block.
1771 // If a spill is needed for the pc register pair, we need to insert a spill
1772 // restore block right before the destination block, and insert a short branch
1773 // into the old destination block's fallthrough predecessor.
1774 // e.g.:
1776 // s_cbranch_scc0 skip_long_branch:
1778 // long_branch_bb:
1779 // spill s[8:9]
1780 // s_getpc_b64 s[8:9]
1781 // s_add_u32 s8, s8, restore_bb
1782 // s_addc_u32 s9, s9, 0
1783 // s_setpc_b64 s[8:9]
1785 // skip_long_branch:
1786 // foo;
1788 // .....
1790 // dest_bb_fallthrough_predecessor:
1791 // bar;
1792 // s_branch dest_bb
1794 // restore_bb:
1795 // restore s[8:9]
1796 // fallthrough dest_bb
1798 // dest_bb:
1799 // buzz;
1801 RS->enterBasicBlockEnd(MBB);
1802 unsigned Scav = RS->scavengeRegisterBackwards(
1803 AMDGPU::SReg_64RegClass,
1804 MachineBasicBlock::iterator(GetPC), false, 0);
1805 MRI.replaceRegWith(PCReg, Scav);
1806 MRI.clearVirtRegs();
1807 RS->setRegUsed(Scav);
1809 return 4 + 8 + 4 + 4;
1812 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1813 switch (Cond) {
1814 case SIInstrInfo::SCC_TRUE:
1815 return AMDGPU::S_CBRANCH_SCC1;
1816 case SIInstrInfo::SCC_FALSE:
1817 return AMDGPU::S_CBRANCH_SCC0;
1818 case SIInstrInfo::VCCNZ:
1819 return AMDGPU::S_CBRANCH_VCCNZ;
1820 case SIInstrInfo::VCCZ:
1821 return AMDGPU::S_CBRANCH_VCCZ;
1822 case SIInstrInfo::EXECNZ:
1823 return AMDGPU::S_CBRANCH_EXECNZ;
1824 case SIInstrInfo::EXECZ:
1825 return AMDGPU::S_CBRANCH_EXECZ;
1826 default:
1827 llvm_unreachable("invalid branch predicate");
1831 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1832 switch (Opcode) {
1833 case AMDGPU::S_CBRANCH_SCC0:
1834 return SCC_FALSE;
1835 case AMDGPU::S_CBRANCH_SCC1:
1836 return SCC_TRUE;
1837 case AMDGPU::S_CBRANCH_VCCNZ:
1838 return VCCNZ;
1839 case AMDGPU::S_CBRANCH_VCCZ:
1840 return VCCZ;
1841 case AMDGPU::S_CBRANCH_EXECNZ:
1842 return EXECNZ;
1843 case AMDGPU::S_CBRANCH_EXECZ:
1844 return EXECZ;
1845 default:
1846 return INVALID_BR;
1850 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
1851 MachineBasicBlock::iterator I,
1852 MachineBasicBlock *&TBB,
1853 MachineBasicBlock *&FBB,
1854 SmallVectorImpl<MachineOperand> &Cond,
1855 bool AllowModify) const {
1856 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1857 // Unconditional Branch
1858 TBB = I->getOperand(0).getMBB();
1859 return false;
1862 MachineBasicBlock *CondBB = nullptr;
1864 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1865 CondBB = I->getOperand(1).getMBB();
1866 Cond.push_back(I->getOperand(0));
1867 } else {
1868 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1869 if (Pred == INVALID_BR)
1870 return true;
1872 CondBB = I->getOperand(0).getMBB();
1873 Cond.push_back(MachineOperand::CreateImm(Pred));
1874 Cond.push_back(I->getOperand(1)); // Save the branch register.
1876 ++I;
1878 if (I == MBB.end()) {
1879 // Conditional branch followed by fall-through.
1880 TBB = CondBB;
1881 return false;
1884 if (I->getOpcode() == AMDGPU::S_BRANCH) {
1885 TBB = CondBB;
1886 FBB = I->getOperand(0).getMBB();
1887 return false;
1890 return true;
1893 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
1894 MachineBasicBlock *&FBB,
1895 SmallVectorImpl<MachineOperand> &Cond,
1896 bool AllowModify) const {
1897 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1898 auto E = MBB.end();
1899 if (I == E)
1900 return false;
1902 // Skip over the instructions that are artificially terminators for special
1903 // exec management.
1904 while (I != E && !I->isBranch() && !I->isReturn() &&
1905 I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1906 switch (I->getOpcode()) {
1907 case AMDGPU::SI_MASK_BRANCH:
1908 case AMDGPU::S_MOV_B64_term:
1909 case AMDGPU::S_XOR_B64_term:
1910 case AMDGPU::S_ANDN2_B64_term:
1911 case AMDGPU::S_MOV_B32_term:
1912 case AMDGPU::S_XOR_B32_term:
1913 case AMDGPU::S_OR_B32_term:
1914 case AMDGPU::S_ANDN2_B32_term:
1915 break;
1916 case AMDGPU::SI_IF:
1917 case AMDGPU::SI_ELSE:
1918 case AMDGPU::SI_KILL_I1_TERMINATOR:
1919 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1920 // FIXME: It's messy that these need to be considered here at all.
1921 return true;
1922 default:
1923 llvm_unreachable("unexpected non-branch terminator inst");
1926 ++I;
1929 if (I == E)
1930 return false;
1932 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1933 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1935 ++I;
1937 // TODO: Should be able to treat as fallthrough?
1938 if (I == MBB.end())
1939 return true;
1941 if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1942 return true;
1944 MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1946 // Specifically handle the case where the conditional branch is to the same
1947 // destination as the mask branch. e.g.
1949 // si_mask_branch BB8
1950 // s_cbranch_execz BB8
1951 // s_cbranch BB9
1953 // This is required to understand divergent loops which may need the branches
1954 // to be relaxed.
1955 if (TBB != MaskBrDest || Cond.empty())
1956 return true;
1958 auto Pred = Cond[0].getImm();
1959 return (Pred != EXECZ && Pred != EXECNZ);
1962 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
1963 int *BytesRemoved) const {
1964 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1966 unsigned Count = 0;
1967 unsigned RemovedSize = 0;
1968 while (I != MBB.end()) {
1969 MachineBasicBlock::iterator Next = std::next(I);
1970 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1971 I = Next;
1972 continue;
1975 RemovedSize += getInstSizeInBytes(*I);
1976 I->eraseFromParent();
1977 ++Count;
1978 I = Next;
1981 if (BytesRemoved)
1982 *BytesRemoved = RemovedSize;
1984 return Count;
1987 // Copy the flags onto the implicit condition register operand.
1988 static void preserveCondRegFlags(MachineOperand &CondReg,
1989 const MachineOperand &OrigCond) {
1990 CondReg.setIsUndef(OrigCond.isUndef());
1991 CondReg.setIsKill(OrigCond.isKill());
1994 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
1995 MachineBasicBlock *TBB,
1996 MachineBasicBlock *FBB,
1997 ArrayRef<MachineOperand> Cond,
1998 const DebugLoc &DL,
1999 int *BytesAdded) const {
2000 if (!FBB && Cond.empty()) {
2001 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2002 .addMBB(TBB);
2003 if (BytesAdded)
2004 *BytesAdded = 4;
2005 return 1;
2008 if(Cond.size() == 1 && Cond[0].isReg()) {
2009 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2010 .add(Cond[0])
2011 .addMBB(TBB);
2012 return 1;
2015 assert(TBB && Cond[0].isImm());
2017 unsigned Opcode
2018 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2020 if (!FBB) {
2021 Cond[1].isUndef();
2022 MachineInstr *CondBr =
2023 BuildMI(&MBB, DL, get(Opcode))
2024 .addMBB(TBB);
2026 // Copy the flags onto the implicit condition register operand.
2027 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2029 if (BytesAdded)
2030 *BytesAdded = 4;
2031 return 1;
2034 assert(TBB && FBB);
2036 MachineInstr *CondBr =
2037 BuildMI(&MBB, DL, get(Opcode))
2038 .addMBB(TBB);
2039 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2040 .addMBB(FBB);
2042 MachineOperand &CondReg = CondBr->getOperand(1);
2043 CondReg.setIsUndef(Cond[1].isUndef());
2044 CondReg.setIsKill(Cond[1].isKill());
2046 if (BytesAdded)
2047 *BytesAdded = 8;
2049 return 2;
2052 bool SIInstrInfo::reverseBranchCondition(
2053 SmallVectorImpl<MachineOperand> &Cond) const {
2054 if (Cond.size() != 2) {
2055 return true;
2058 if (Cond[0].isImm()) {
2059 Cond[0].setImm(-Cond[0].getImm());
2060 return false;
2063 return true;
2066 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
2067 ArrayRef<MachineOperand> Cond,
2068 unsigned TrueReg, unsigned FalseReg,
2069 int &CondCycles,
2070 int &TrueCycles, int &FalseCycles) const {
2071 switch (Cond[0].getImm()) {
2072 case VCCNZ:
2073 case VCCZ: {
2074 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2075 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2076 assert(MRI.getRegClass(FalseReg) == RC);
2078 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2079 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2081 // Limit to equal cost for branch vs. N v_cndmask_b32s.
2082 return RI.hasVGPRs(RC) && NumInsts <= 6;
2084 case SCC_TRUE:
2085 case SCC_FALSE: {
2086 // FIXME: We could insert for VGPRs if we could replace the original compare
2087 // with a vector one.
2088 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2089 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2090 assert(MRI.getRegClass(FalseReg) == RC);
2092 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2094 // Multiples of 8 can do s_cselect_b64
2095 if (NumInsts % 2 == 0)
2096 NumInsts /= 2;
2098 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2099 return RI.isSGPRClass(RC);
2101 default:
2102 return false;
2106 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
2107 MachineBasicBlock::iterator I, const DebugLoc &DL,
2108 unsigned DstReg, ArrayRef<MachineOperand> Cond,
2109 unsigned TrueReg, unsigned FalseReg) const {
2110 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2111 if (Pred == VCCZ || Pred == SCC_FALSE) {
2112 Pred = static_cast<BranchPredicate>(-Pred);
2113 std::swap(TrueReg, FalseReg);
2116 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2117 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2118 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2120 if (DstSize == 32) {
2121 unsigned SelOp = Pred == SCC_TRUE ?
2122 AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2124 // Instruction's operands are backwards from what is expected.
2125 MachineInstr *Select =
2126 BuildMI(MBB, I, DL, get(SelOp), DstReg)
2127 .addReg(FalseReg)
2128 .addReg(TrueReg);
2130 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2131 return;
2134 if (DstSize == 64 && Pred == SCC_TRUE) {
2135 MachineInstr *Select =
2136 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2137 .addReg(FalseReg)
2138 .addReg(TrueReg);
2140 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2141 return;
2144 static const int16_t Sub0_15[] = {
2145 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2146 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2147 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2148 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2151 static const int16_t Sub0_15_64[] = {
2152 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2153 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2154 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2155 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2158 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2159 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2160 const int16_t *SubIndices = Sub0_15;
2161 int NElts = DstSize / 32;
2163 // 64-bit select is only available for SALU.
2164 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2165 if (Pred == SCC_TRUE) {
2166 if (NElts % 2) {
2167 SelOp = AMDGPU::S_CSELECT_B32;
2168 EltRC = &AMDGPU::SGPR_32RegClass;
2169 } else {
2170 SelOp = AMDGPU::S_CSELECT_B64;
2171 EltRC = &AMDGPU::SGPR_64RegClass;
2172 SubIndices = Sub0_15_64;
2173 NElts /= 2;
2177 MachineInstrBuilder MIB = BuildMI(
2178 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2180 I = MIB->getIterator();
2182 SmallVector<unsigned, 8> Regs;
2183 for (int Idx = 0; Idx != NElts; ++Idx) {
2184 Register DstElt = MRI.createVirtualRegister(EltRC);
2185 Regs.push_back(DstElt);
2187 unsigned SubIdx = SubIndices[Idx];
2189 MachineInstr *Select =
2190 BuildMI(MBB, I, DL, get(SelOp), DstElt)
2191 .addReg(FalseReg, 0, SubIdx)
2192 .addReg(TrueReg, 0, SubIdx);
2193 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2194 fixImplicitOperands(*Select);
2196 MIB.addReg(DstElt)
2197 .addImm(SubIdx);
2201 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const {
2202 switch (MI.getOpcode()) {
2203 case AMDGPU::V_MOV_B32_e32:
2204 case AMDGPU::V_MOV_B32_e64:
2205 case AMDGPU::V_MOV_B64_PSEUDO: {
2206 // If there are additional implicit register operands, this may be used for
2207 // register indexing so the source register operand isn't simply copied.
2208 unsigned NumOps = MI.getDesc().getNumOperands() +
2209 MI.getDesc().getNumImplicitUses();
2211 return MI.getNumOperands() == NumOps;
2213 case AMDGPU::S_MOV_B32:
2214 case AMDGPU::S_MOV_B64:
2215 case AMDGPU::COPY:
2216 case AMDGPU::V_ACCVGPR_WRITE_B32:
2217 case AMDGPU::V_ACCVGPR_READ_B32:
2218 return true;
2219 default:
2220 return false;
2224 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
2225 unsigned Kind) const {
2226 switch(Kind) {
2227 case PseudoSourceValue::Stack:
2228 case PseudoSourceValue::FixedStack:
2229 return AMDGPUAS::PRIVATE_ADDRESS;
2230 case PseudoSourceValue::ConstantPool:
2231 case PseudoSourceValue::GOT:
2232 case PseudoSourceValue::JumpTable:
2233 case PseudoSourceValue::GlobalValueCallEntry:
2234 case PseudoSourceValue::ExternalSymbolCallEntry:
2235 case PseudoSourceValue::TargetCustom:
2236 return AMDGPUAS::CONSTANT_ADDRESS;
2238 return AMDGPUAS::FLAT_ADDRESS;
2241 static void removeModOperands(MachineInstr &MI) {
2242 unsigned Opc = MI.getOpcode();
2243 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2244 AMDGPU::OpName::src0_modifiers);
2245 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2246 AMDGPU::OpName::src1_modifiers);
2247 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2248 AMDGPU::OpName::src2_modifiers);
2250 MI.RemoveOperand(Src2ModIdx);
2251 MI.RemoveOperand(Src1ModIdx);
2252 MI.RemoveOperand(Src0ModIdx);
2255 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2256 unsigned Reg, MachineRegisterInfo *MRI) const {
2257 if (!MRI->hasOneNonDBGUse(Reg))
2258 return false;
2260 switch (DefMI.getOpcode()) {
2261 default:
2262 return false;
2263 case AMDGPU::S_MOV_B64:
2264 // TODO: We could fold 64-bit immediates, but this get compilicated
2265 // when there are sub-registers.
2266 return false;
2268 case AMDGPU::V_MOV_B32_e32:
2269 case AMDGPU::S_MOV_B32:
2270 case AMDGPU::V_ACCVGPR_WRITE_B32:
2271 break;
2274 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2275 assert(ImmOp);
2276 // FIXME: We could handle FrameIndex values here.
2277 if (!ImmOp->isImm())
2278 return false;
2280 unsigned Opc = UseMI.getOpcode();
2281 if (Opc == AMDGPU::COPY) {
2282 bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2283 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2284 if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2285 if (!isInlineConstant(*ImmOp, AMDGPU::OPERAND_REG_INLINE_AC_INT32))
2286 return false;
2287 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2289 UseMI.setDesc(get(NewOpc));
2290 UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2291 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2292 return true;
2295 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2296 Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2297 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2298 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2299 // Don't fold if we are using source or output modifiers. The new VOP2
2300 // instructions don't have them.
2301 if (hasAnyModifiersSet(UseMI))
2302 return false;
2304 // If this is a free constant, there's no reason to do this.
2305 // TODO: We could fold this here instead of letting SIFoldOperands do it
2306 // later.
2307 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2309 // Any src operand can be used for the legality check.
2310 if (isInlineConstant(UseMI, *Src0, *ImmOp))
2311 return false;
2313 bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2314 Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2315 bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2316 Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2317 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2318 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2320 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2321 // We should only expect these to be on src0 due to canonicalizations.
2322 if (Src0->isReg() && Src0->getReg() == Reg) {
2323 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2324 return false;
2326 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2327 return false;
2329 unsigned NewOpc =
2330 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2331 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2332 if (pseudoToMCOpcode(NewOpc) == -1)
2333 return false;
2335 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2337 const int64_t Imm = ImmOp->getImm();
2339 // FIXME: This would be a lot easier if we could return a new instruction
2340 // instead of having to modify in place.
2342 // Remove these first since they are at the end.
2343 UseMI.RemoveOperand(
2344 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2345 UseMI.RemoveOperand(
2346 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2348 Register Src1Reg = Src1->getReg();
2349 unsigned Src1SubReg = Src1->getSubReg();
2350 Src0->setReg(Src1Reg);
2351 Src0->setSubReg(Src1SubReg);
2352 Src0->setIsKill(Src1->isKill());
2354 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2355 Opc == AMDGPU::V_MAC_F16_e64 ||
2356 Opc == AMDGPU::V_FMAC_F32_e64 ||
2357 Opc == AMDGPU::V_FMAC_F16_e64)
2358 UseMI.untieRegOperand(
2359 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2361 Src1->ChangeToImmediate(Imm);
2363 removeModOperands(UseMI);
2364 UseMI.setDesc(get(NewOpc));
2366 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2367 if (DeleteDef)
2368 DefMI.eraseFromParent();
2370 return true;
2373 // Added part is the constant: Use v_madak_{f16, f32}.
2374 if (Src2->isReg() && Src2->getReg() == Reg) {
2375 // Not allowed to use constant bus for another operand.
2376 // We can however allow an inline immediate as src0.
2377 bool Src0Inlined = false;
2378 if (Src0->isReg()) {
2379 // Try to inline constant if possible.
2380 // If the Def moves immediate and the use is single
2381 // We are saving VGPR here.
2382 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2383 if (Def && Def->isMoveImmediate() &&
2384 isInlineConstant(Def->getOperand(1)) &&
2385 MRI->hasOneUse(Src0->getReg())) {
2386 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2387 Src0Inlined = true;
2388 } else if ((Register::isPhysicalRegister(Src0->getReg()) &&
2389 (ST.getConstantBusLimit(Opc) <= 1 &&
2390 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2391 (Register::isVirtualRegister(Src0->getReg()) &&
2392 (ST.getConstantBusLimit(Opc) <= 1 &&
2393 RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2394 return false;
2395 // VGPR is okay as Src0 - fallthrough
2398 if (Src1->isReg() && !Src0Inlined ) {
2399 // We have one slot for inlinable constant so far - try to fill it
2400 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2401 if (Def && Def->isMoveImmediate() &&
2402 isInlineConstant(Def->getOperand(1)) &&
2403 MRI->hasOneUse(Src1->getReg()) &&
2404 commuteInstruction(UseMI)) {
2405 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2406 } else if ((Register::isPhysicalRegister(Src1->getReg()) &&
2407 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2408 (Register::isVirtualRegister(Src1->getReg()) &&
2409 RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2410 return false;
2411 // VGPR is okay as Src1 - fallthrough
2414 unsigned NewOpc =
2415 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2416 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2417 if (pseudoToMCOpcode(NewOpc) == -1)
2418 return false;
2420 const int64_t Imm = ImmOp->getImm();
2422 // FIXME: This would be a lot easier if we could return a new instruction
2423 // instead of having to modify in place.
2425 // Remove these first since they are at the end.
2426 UseMI.RemoveOperand(
2427 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2428 UseMI.RemoveOperand(
2429 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2431 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2432 Opc == AMDGPU::V_MAC_F16_e64 ||
2433 Opc == AMDGPU::V_FMAC_F32_e64 ||
2434 Opc == AMDGPU::V_FMAC_F16_e64)
2435 UseMI.untieRegOperand(
2436 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2438 // ChangingToImmediate adds Src2 back to the instruction.
2439 Src2->ChangeToImmediate(Imm);
2441 // These come before src2.
2442 removeModOperands(UseMI);
2443 UseMI.setDesc(get(NewOpc));
2444 // It might happen that UseMI was commuted
2445 // and we now have SGPR as SRC1. If so 2 inlined
2446 // constant and SGPR are illegal.
2447 legalizeOperands(UseMI);
2449 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2450 if (DeleteDef)
2451 DefMI.eraseFromParent();
2453 return true;
2457 return false;
2460 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2461 int WidthB, int OffsetB) {
2462 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2463 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2464 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2465 return LowOffset + LowWidth <= HighOffset;
2468 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2469 const MachineInstr &MIb) const {
2470 const MachineOperand *BaseOp0, *BaseOp1;
2471 int64_t Offset0, Offset1;
2473 if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2474 getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2475 if (!BaseOp0->isIdenticalTo(*BaseOp1))
2476 return false;
2478 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2479 // FIXME: Handle ds_read2 / ds_write2.
2480 return false;
2482 unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2483 unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2484 if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2485 return true;
2489 return false;
2492 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
2493 const MachineInstr &MIb) const {
2494 assert((MIa.mayLoad() || MIa.mayStore()) &&
2495 "MIa must load from or modify a memory location");
2496 assert((MIb.mayLoad() || MIb.mayStore()) &&
2497 "MIb must load from or modify a memory location");
2499 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
2500 return false;
2502 // XXX - Can we relax this between address spaces?
2503 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2504 return false;
2506 // TODO: Should we check the address space from the MachineMemOperand? That
2507 // would allow us to distinguish objects we know don't alias based on the
2508 // underlying address space, even if it was lowered to a different one,
2509 // e.g. private accesses lowered to use MUBUF instructions on a scratch
2510 // buffer.
2511 if (isDS(MIa)) {
2512 if (isDS(MIb))
2513 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2515 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2518 if (isMUBUF(MIa) || isMTBUF(MIa)) {
2519 if (isMUBUF(MIb) || isMTBUF(MIb))
2520 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2522 return !isFLAT(MIb) && !isSMRD(MIb);
2525 if (isSMRD(MIa)) {
2526 if (isSMRD(MIb))
2527 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2529 return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2532 if (isFLAT(MIa)) {
2533 if (isFLAT(MIb))
2534 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2536 return false;
2539 return false;
2542 static int64_t getFoldableImm(const MachineOperand* MO) {
2543 if (!MO->isReg())
2544 return false;
2545 const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2546 const MachineRegisterInfo &MRI = MF->getRegInfo();
2547 auto Def = MRI.getUniqueVRegDef(MO->getReg());
2548 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2549 Def->getOperand(1).isImm())
2550 return Def->getOperand(1).getImm();
2551 return AMDGPU::NoRegister;
2554 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
2555 MachineInstr &MI,
2556 LiveVariables *LV) const {
2557 unsigned Opc = MI.getOpcode();
2558 bool IsF16 = false;
2559 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2560 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2562 switch (Opc) {
2563 default:
2564 return nullptr;
2565 case AMDGPU::V_MAC_F16_e64:
2566 case AMDGPU::V_FMAC_F16_e64:
2567 IsF16 = true;
2568 LLVM_FALLTHROUGH;
2569 case AMDGPU::V_MAC_F32_e64:
2570 case AMDGPU::V_FMAC_F32_e64:
2571 break;
2572 case AMDGPU::V_MAC_F16_e32:
2573 case AMDGPU::V_FMAC_F16_e32:
2574 IsF16 = true;
2575 LLVM_FALLTHROUGH;
2576 case AMDGPU::V_MAC_F32_e32:
2577 case AMDGPU::V_FMAC_F32_e32: {
2578 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2579 AMDGPU::OpName::src0);
2580 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2581 if (!Src0->isReg() && !Src0->isImm())
2582 return nullptr;
2584 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2585 return nullptr;
2587 break;
2591 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2592 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2593 const MachineOperand *Src0Mods =
2594 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2595 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2596 const MachineOperand *Src1Mods =
2597 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2598 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2599 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2600 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2602 if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2603 // If we have an SGPR input, we will violate the constant bus restriction.
2604 (ST.getConstantBusLimit(Opc) > 1 ||
2605 !Src0->isReg() ||
2606 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2607 if (auto Imm = getFoldableImm(Src2)) {
2608 unsigned NewOpc =
2609 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2610 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2611 if (pseudoToMCOpcode(NewOpc) != -1)
2612 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2613 .add(*Dst)
2614 .add(*Src0)
2615 .add(*Src1)
2616 .addImm(Imm);
2618 unsigned NewOpc =
2619 IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2620 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2621 if (auto Imm = getFoldableImm(Src1)) {
2622 if (pseudoToMCOpcode(NewOpc) != -1)
2623 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2624 .add(*Dst)
2625 .add(*Src0)
2626 .addImm(Imm)
2627 .add(*Src2);
2629 if (auto Imm = getFoldableImm(Src0)) {
2630 if (pseudoToMCOpcode(NewOpc) != -1 &&
2631 isOperandLegal(MI, AMDGPU::getNamedOperandIdx(NewOpc,
2632 AMDGPU::OpName::src0), Src1))
2633 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2634 .add(*Dst)
2635 .add(*Src1)
2636 .addImm(Imm)
2637 .add(*Src2);
2641 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2642 : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2643 if (pseudoToMCOpcode(NewOpc) == -1)
2644 return nullptr;
2646 return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2647 .add(*Dst)
2648 .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2649 .add(*Src0)
2650 .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2651 .add(*Src1)
2652 .addImm(0) // Src mods
2653 .add(*Src2)
2654 .addImm(Clamp ? Clamp->getImm() : 0)
2655 .addImm(Omod ? Omod->getImm() : 0);
2658 // It's not generally safe to move VALU instructions across these since it will
2659 // start using the register as a base index rather than directly.
2660 // XXX - Why isn't hasSideEffects sufficient for these?
2661 static bool changesVGPRIndexingMode(const MachineInstr &MI) {
2662 switch (MI.getOpcode()) {
2663 case AMDGPU::S_SET_GPR_IDX_ON:
2664 case AMDGPU::S_SET_GPR_IDX_MODE:
2665 case AMDGPU::S_SET_GPR_IDX_OFF:
2666 return true;
2667 default:
2668 return false;
2672 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2673 const MachineBasicBlock *MBB,
2674 const MachineFunction &MF) const {
2675 // XXX - Do we want the SP check in the base implementation?
2677 // Target-independent instructions do not have an implicit-use of EXEC, even
2678 // when they operate on VGPRs. Treating EXEC modifications as scheduling
2679 // boundaries prevents incorrect movements of such instructions.
2680 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2681 MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2682 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2683 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2684 MI.getOpcode() == AMDGPU::S_DENORM_MODE ||
2685 changesVGPRIndexingMode(MI);
2688 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2689 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2690 Opcode == AMDGPU::DS_GWS_INIT ||
2691 Opcode == AMDGPU::DS_GWS_SEMA_V ||
2692 Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2693 Opcode == AMDGPU::DS_GWS_SEMA_P ||
2694 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2695 Opcode == AMDGPU::DS_GWS_BARRIER;
2698 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
2699 unsigned Opcode = MI.getOpcode();
2701 if (MI.mayStore() && isSMRD(MI))
2702 return true; // scalar store or atomic
2704 // This will terminate the function when other lanes may need to continue.
2705 if (MI.isReturn())
2706 return true;
2708 // These instructions cause shader I/O that may cause hardware lockups
2709 // when executed with an empty EXEC mask.
2711 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2712 // EXEC = 0, but checking for that case here seems not worth it
2713 // given the typical code patterns.
2714 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2715 Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2716 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2717 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2718 return true;
2720 if (MI.isCall() || MI.isInlineAsm())
2721 return true; // conservative assumption
2723 // These are like SALU instructions in terms of effects, so it's questionable
2724 // whether we should return true for those.
2726 // However, executing them with EXEC = 0 causes them to operate on undefined
2727 // data, which we avoid by returning true here.
2728 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2729 return true;
2731 return false;
2734 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
2735 const MachineInstr &MI) const {
2736 if (MI.isMetaInstruction())
2737 return false;
2739 // This won't read exec if this is an SGPR->SGPR copy.
2740 if (MI.isCopyLike()) {
2741 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2742 return true;
2744 // Make sure this isn't copying exec as a normal operand
2745 return MI.readsRegister(AMDGPU::EXEC, &RI);
2748 // Make a conservative assumption about the callee.
2749 if (MI.isCall())
2750 return true;
2752 // Be conservative with any unhandled generic opcodes.
2753 if (!isTargetSpecificOpcode(MI.getOpcode()))
2754 return true;
2756 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2759 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2760 switch (Imm.getBitWidth()) {
2761 case 1: // This likely will be a condition code mask.
2762 return true;
2764 case 32:
2765 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
2766 ST.hasInv2PiInlineImm());
2767 case 64:
2768 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
2769 ST.hasInv2PiInlineImm());
2770 case 16:
2771 return ST.has16BitInsts() &&
2772 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
2773 ST.hasInv2PiInlineImm());
2774 default:
2775 llvm_unreachable("invalid bitwidth");
2779 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
2780 uint8_t OperandType) const {
2781 if (!MO.isImm() ||
2782 OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2783 OperandType > AMDGPU::OPERAND_SRC_LAST)
2784 return false;
2786 // MachineOperand provides no way to tell the true operand size, since it only
2787 // records a 64-bit value. We need to know the size to determine if a 32-bit
2788 // floating point immediate bit pattern is legal for an integer immediate. It
2789 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2791 int64_t Imm = MO.getImm();
2792 switch (OperandType) {
2793 case AMDGPU::OPERAND_REG_IMM_INT32:
2794 case AMDGPU::OPERAND_REG_IMM_FP32:
2795 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2796 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2797 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2798 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: {
2799 int32_t Trunc = static_cast<int32_t>(Imm);
2800 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
2802 case AMDGPU::OPERAND_REG_IMM_INT64:
2803 case AMDGPU::OPERAND_REG_IMM_FP64:
2804 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2805 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2806 return AMDGPU::isInlinableLiteral64(MO.getImm(),
2807 ST.hasInv2PiInlineImm());
2808 case AMDGPU::OPERAND_REG_IMM_INT16:
2809 case AMDGPU::OPERAND_REG_IMM_FP16:
2810 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2811 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2812 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
2813 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
2814 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2815 // A few special case instructions have 16-bit operands on subtargets
2816 // where 16-bit instructions are not legal.
2817 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2818 // constants in these cases
2819 int16_t Trunc = static_cast<int16_t>(Imm);
2820 return ST.has16BitInsts() &&
2821 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
2824 return false;
2826 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2827 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2828 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2829 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2830 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
2831 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
2832 uint32_t Trunc = static_cast<uint32_t>(Imm);
2833 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm());
2835 default:
2836 llvm_unreachable("invalid bitwidth");
2840 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO,
2841 const MCOperandInfo &OpInfo) const {
2842 switch (MO.getType()) {
2843 case MachineOperand::MO_Register:
2844 return false;
2845 case MachineOperand::MO_Immediate:
2846 return !isInlineConstant(MO, OpInfo);
2847 case MachineOperand::MO_FrameIndex:
2848 case MachineOperand::MO_MachineBasicBlock:
2849 case MachineOperand::MO_ExternalSymbol:
2850 case MachineOperand::MO_GlobalAddress:
2851 case MachineOperand::MO_MCSymbol:
2852 return true;
2853 default:
2854 llvm_unreachable("unexpected operand type");
2858 static bool compareMachineOp(const MachineOperand &Op0,
2859 const MachineOperand &Op1) {
2860 if (Op0.getType() != Op1.getType())
2861 return false;
2863 switch (Op0.getType()) {
2864 case MachineOperand::MO_Register:
2865 return Op0.getReg() == Op1.getReg();
2866 case MachineOperand::MO_Immediate:
2867 return Op0.getImm() == Op1.getImm();
2868 default:
2869 llvm_unreachable("Didn't expect to be comparing these operand types");
2873 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
2874 const MachineOperand &MO) const {
2875 const MCInstrDesc &InstDesc = MI.getDesc();
2876 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2878 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2880 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2881 return true;
2883 if (OpInfo.RegClass < 0)
2884 return false;
2886 const MachineFunction *MF = MI.getParent()->getParent();
2887 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2889 if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
2890 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
2891 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2892 AMDGPU::OpName::src2))
2893 return false;
2894 return RI.opCanUseInlineConstant(OpInfo.OperandType);
2897 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2898 return false;
2900 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2901 return true;
2903 return ST.hasVOP3Literal();
2906 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2907 int Op32 = AMDGPU::getVOPe32(Opcode);
2908 if (Op32 == -1)
2909 return false;
2911 return pseudoToMCOpcode(Op32) != -1;
2914 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2915 // The src0_modifier operand is present on all instructions
2916 // that have modifiers.
2918 return AMDGPU::getNamedOperandIdx(Opcode,
2919 AMDGPU::OpName::src0_modifiers) != -1;
2922 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
2923 unsigned OpName) const {
2924 const MachineOperand *Mods = getNamedOperand(MI, OpName);
2925 return Mods && Mods->getImm();
2928 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
2929 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2930 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2931 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2932 hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2933 hasModifiersSet(MI, AMDGPU::OpName::omod);
2936 bool SIInstrInfo::canShrink(const MachineInstr &MI,
2937 const MachineRegisterInfo &MRI) const {
2938 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2939 // Can't shrink instruction with three operands.
2940 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2941 // a special case for it. It can only be shrunk if the third operand
2942 // is vcc, and src0_modifiers and src1_modifiers are not set.
2943 // We should handle this the same way we handle vopc, by addding
2944 // a register allocation hint pre-regalloc and then do the shrinking
2945 // post-regalloc.
2946 if (Src2) {
2947 switch (MI.getOpcode()) {
2948 default: return false;
2950 case AMDGPU::V_ADDC_U32_e64:
2951 case AMDGPU::V_SUBB_U32_e64:
2952 case AMDGPU::V_SUBBREV_U32_e64: {
2953 const MachineOperand *Src1
2954 = getNamedOperand(MI, AMDGPU::OpName::src1);
2955 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2956 return false;
2957 // Additional verification is needed for sdst/src2.
2958 return true;
2960 case AMDGPU::V_MAC_F32_e64:
2961 case AMDGPU::V_MAC_F16_e64:
2962 case AMDGPU::V_FMAC_F32_e64:
2963 case AMDGPU::V_FMAC_F16_e64:
2964 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2965 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2966 return false;
2967 break;
2969 case AMDGPU::V_CNDMASK_B32_e64:
2970 break;
2974 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2975 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2976 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2977 return false;
2979 // We don't need to check src0, all input types are legal, so just make sure
2980 // src0 isn't using any modifiers.
2981 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2982 return false;
2984 // Can it be shrunk to a valid 32 bit opcode?
2985 if (!hasVALU32BitEncoding(MI.getOpcode()))
2986 return false;
2988 // Check output modifiers
2989 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2990 !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2993 // Set VCC operand with all flags from \p Orig, except for setting it as
2994 // implicit.
2995 static void copyFlagsToImplicitVCC(MachineInstr &MI,
2996 const MachineOperand &Orig) {
2998 for (MachineOperand &Use : MI.implicit_operands()) {
2999 if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
3000 Use.setIsUndef(Orig.isUndef());
3001 Use.setIsKill(Orig.isKill());
3002 return;
3007 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
3008 unsigned Op32) const {
3009 MachineBasicBlock *MBB = MI.getParent();;
3010 MachineInstrBuilder Inst32 =
3011 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
3013 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3014 // For VOPC instructions, this is replaced by an implicit def of vcc.
3015 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3016 if (Op32DstIdx != -1) {
3017 // dst
3018 Inst32.add(MI.getOperand(0));
3019 } else {
3020 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3021 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3022 "Unexpected case");
3025 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3027 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3028 if (Src1)
3029 Inst32.add(*Src1);
3031 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3033 if (Src2) {
3034 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3035 if (Op32Src2Idx != -1) {
3036 Inst32.add(*Src2);
3037 } else {
3038 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3039 // replaced with an implicit read of vcc. This was already added
3040 // during the initial BuildMI, so find it to preserve the flags.
3041 copyFlagsToImplicitVCC(*Inst32, *Src2);
3045 return Inst32;
3048 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
3049 const MachineOperand &MO,
3050 const MCOperandInfo &OpInfo) const {
3051 // Literal constants use the constant bus.
3052 //if (isLiteralConstantLike(MO, OpInfo))
3053 // return true;
3054 if (MO.isImm())
3055 return !isInlineConstant(MO, OpInfo);
3057 if (!MO.isReg())
3058 return true; // Misc other operands like FrameIndex
3060 if (!MO.isUse())
3061 return false;
3063 if (Register::isVirtualRegister(MO.getReg()))
3064 return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3066 // Null is free
3067 if (MO.getReg() == AMDGPU::SGPR_NULL)
3068 return false;
3070 // SGPRs use the constant bus
3071 if (MO.isImplicit()) {
3072 return MO.getReg() == AMDGPU::M0 ||
3073 MO.getReg() == AMDGPU::VCC ||
3074 MO.getReg() == AMDGPU::VCC_LO;
3075 } else {
3076 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3077 AMDGPU::SReg_64RegClass.contains(MO.getReg());
3081 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3082 for (const MachineOperand &MO : MI.implicit_operands()) {
3083 // We only care about reads.
3084 if (MO.isDef())
3085 continue;
3087 switch (MO.getReg()) {
3088 case AMDGPU::VCC:
3089 case AMDGPU::VCC_LO:
3090 case AMDGPU::VCC_HI:
3091 case AMDGPU::M0:
3092 case AMDGPU::FLAT_SCR:
3093 return MO.getReg();
3095 default:
3096 break;
3100 return AMDGPU::NoRegister;
3103 static bool shouldReadExec(const MachineInstr &MI) {
3104 if (SIInstrInfo::isVALU(MI)) {
3105 switch (MI.getOpcode()) {
3106 case AMDGPU::V_READLANE_B32:
3107 case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3108 case AMDGPU::V_READLANE_B32_gfx10:
3109 case AMDGPU::V_READLANE_B32_vi:
3110 case AMDGPU::V_WRITELANE_B32:
3111 case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3112 case AMDGPU::V_WRITELANE_B32_gfx10:
3113 case AMDGPU::V_WRITELANE_B32_vi:
3114 return false;
3117 return true;
3120 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3121 SIInstrInfo::isSALU(MI) ||
3122 SIInstrInfo::isSMRD(MI))
3123 return false;
3125 return true;
3128 static bool isSubRegOf(const SIRegisterInfo &TRI,
3129 const MachineOperand &SuperVec,
3130 const MachineOperand &SubReg) {
3131 if (Register::isPhysicalRegister(SubReg.getReg()))
3132 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3134 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3135 SubReg.getReg() == SuperVec.getReg();
3138 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
3139 StringRef &ErrInfo) const {
3140 uint16_t Opcode = MI.getOpcode();
3141 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3142 return true;
3144 const MachineFunction *MF = MI.getParent()->getParent();
3145 const MachineRegisterInfo &MRI = MF->getRegInfo();
3147 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3148 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3149 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3151 // Make sure the number of operands is correct.
3152 const MCInstrDesc &Desc = get(Opcode);
3153 if (!Desc.isVariadic() &&
3154 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3155 ErrInfo = "Instruction has wrong number of operands.";
3156 return false;
3159 if (MI.isInlineAsm()) {
3160 // Verify register classes for inlineasm constraints.
3161 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3162 I != E; ++I) {
3163 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3164 if (!RC)
3165 continue;
3167 const MachineOperand &Op = MI.getOperand(I);
3168 if (!Op.isReg())
3169 continue;
3171 Register Reg = Op.getReg();
3172 if (!Register::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3173 ErrInfo = "inlineasm operand has incorrect register class.";
3174 return false;
3178 return true;
3181 // Make sure the register classes are correct.
3182 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3183 if (MI.getOperand(i).isFPImm()) {
3184 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3185 "all fp values to integers.";
3186 return false;
3189 int RegClass = Desc.OpInfo[i].RegClass;
3191 switch (Desc.OpInfo[i].OperandType) {
3192 case MCOI::OPERAND_REGISTER:
3193 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3194 ErrInfo = "Illegal immediate value for operand.";
3195 return false;
3197 break;
3198 case AMDGPU::OPERAND_REG_IMM_INT32:
3199 case AMDGPU::OPERAND_REG_IMM_FP32:
3200 break;
3201 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
3202 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
3203 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
3204 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
3205 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
3206 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
3207 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
3208 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
3209 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
3210 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
3211 const MachineOperand &MO = MI.getOperand(i);
3212 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3213 ErrInfo = "Illegal immediate value for operand.";
3214 return false;
3216 break;
3218 case MCOI::OPERAND_IMMEDIATE:
3219 case AMDGPU::OPERAND_KIMM32:
3220 // Check if this operand is an immediate.
3221 // FrameIndex operands will be replaced by immediates, so they are
3222 // allowed.
3223 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3224 ErrInfo = "Expected immediate, but got non-immediate";
3225 return false;
3227 LLVM_FALLTHROUGH;
3228 default:
3229 continue;
3232 if (!MI.getOperand(i).isReg())
3233 continue;
3235 if (RegClass != -1) {
3236 Register Reg = MI.getOperand(i).getReg();
3237 if (Reg == AMDGPU::NoRegister || Register::isVirtualRegister(Reg))
3238 continue;
3240 const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3241 if (!RC->contains(Reg)) {
3242 ErrInfo = "Operand has incorrect register class.";
3243 return false;
3248 // Verify SDWA
3249 if (isSDWA(MI)) {
3250 if (!ST.hasSDWA()) {
3251 ErrInfo = "SDWA is not supported on this target";
3252 return false;
3255 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3257 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3259 for (int OpIdx: OpIndicies) {
3260 if (OpIdx == -1)
3261 continue;
3262 const MachineOperand &MO = MI.getOperand(OpIdx);
3264 if (!ST.hasSDWAScalar()) {
3265 // Only VGPRS on VI
3266 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3267 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3268 return false;
3270 } else {
3271 // No immediates on GFX9
3272 if (!MO.isReg()) {
3273 ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3274 return false;
3279 if (!ST.hasSDWAOmod()) {
3280 // No omod allowed on VI
3281 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3282 if (OMod != nullptr &&
3283 (!OMod->isImm() || OMod->getImm() != 0)) {
3284 ErrInfo = "OMod not allowed in SDWA instructions on VI";
3285 return false;
3289 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3290 if (isVOPC(BasicOpcode)) {
3291 if (!ST.hasSDWASdst() && DstIdx != -1) {
3292 // Only vcc allowed as dst on VI for VOPC
3293 const MachineOperand &Dst = MI.getOperand(DstIdx);
3294 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3295 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3296 return false;
3298 } else if (!ST.hasSDWAOutModsVOPC()) {
3299 // No clamp allowed on GFX9 for VOPC
3300 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3301 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3302 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3303 return false;
3306 // No omod allowed on GFX9 for VOPC
3307 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3308 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3309 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3310 return false;
3315 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3316 if (DstUnused && DstUnused->isImm() &&
3317 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3318 const MachineOperand &Dst = MI.getOperand(DstIdx);
3319 if (!Dst.isReg() || !Dst.isTied()) {
3320 ErrInfo = "Dst register should have tied register";
3321 return false;
3324 const MachineOperand &TiedMO =
3325 MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3326 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3327 ErrInfo =
3328 "Dst register should be tied to implicit use of preserved register";
3329 return false;
3330 } else if (Register::isPhysicalRegister(TiedMO.getReg()) &&
3331 Dst.getReg() != TiedMO.getReg()) {
3332 ErrInfo = "Dst register should use same physical register as preserved";
3333 return false;
3338 // Verify MIMG
3339 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3340 // Ensure that the return type used is large enough for all the options
3341 // being used TFE/LWE require an extra result register.
3342 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3343 if (DMask) {
3344 uint64_t DMaskImm = DMask->getImm();
3345 uint32_t RegCount =
3346 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3347 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3348 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3349 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3351 // Adjust for packed 16 bit values
3352 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3353 RegCount >>= 1;
3355 // Adjust if using LWE or TFE
3356 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3357 RegCount += 1;
3359 const uint32_t DstIdx =
3360 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3361 const MachineOperand &Dst = MI.getOperand(DstIdx);
3362 if (Dst.isReg()) {
3363 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3364 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3365 if (RegCount > DstSize) {
3366 ErrInfo = "MIMG instruction returns too many registers for dst "
3367 "register class";
3368 return false;
3374 // Verify VOP*. Ignore multiple sgpr operands on writelane.
3375 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3376 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3377 // Only look at the true operands. Only a real operand can use the constant
3378 // bus, and we don't want to check pseudo-operands like the source modifier
3379 // flags.
3380 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3382 unsigned ConstantBusCount = 0;
3383 unsigned LiteralCount = 0;
3385 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3386 ++ConstantBusCount;
3388 SmallVector<unsigned, 2> SGPRsUsed;
3389 unsigned SGPRUsed = findImplicitSGPRRead(MI);
3390 if (SGPRUsed != AMDGPU::NoRegister) {
3391 ++ConstantBusCount;
3392 SGPRsUsed.push_back(SGPRUsed);
3395 for (int OpIdx : OpIndices) {
3396 if (OpIdx == -1)
3397 break;
3398 const MachineOperand &MO = MI.getOperand(OpIdx);
3399 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3400 if (MO.isReg()) {
3401 SGPRUsed = MO.getReg();
3402 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3403 return !RI.regsOverlap(SGPRUsed, SGPR);
3404 })) {
3405 ++ConstantBusCount;
3406 SGPRsUsed.push_back(SGPRUsed);
3408 } else {
3409 ++ConstantBusCount;
3410 ++LiteralCount;
3414 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3415 // v_writelane_b32 is an exception from constant bus restriction:
3416 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3417 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3418 Opcode != AMDGPU::V_WRITELANE_B32) {
3419 ErrInfo = "VOP* instruction violates constant bus restriction";
3420 return false;
3423 if (isVOP3(MI) && LiteralCount) {
3424 if (LiteralCount && !ST.hasVOP3Literal()) {
3425 ErrInfo = "VOP3 instruction uses literal";
3426 return false;
3428 if (LiteralCount > 1) {
3429 ErrInfo = "VOP3 instruction uses more than one literal";
3430 return false;
3435 // Verify misc. restrictions on specific instructions.
3436 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3437 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3438 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3439 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3440 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3441 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3442 if (!compareMachineOp(Src0, Src1) &&
3443 !compareMachineOp(Src0, Src2)) {
3444 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3445 return false;
3450 if (isSOP2(MI) || isSOPC(MI)) {
3451 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3452 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3453 unsigned Immediates = 0;
3455 if (!Src0.isReg() &&
3456 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3457 Immediates++;
3458 if (!Src1.isReg() &&
3459 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3460 Immediates++;
3462 if (Immediates > 1) {
3463 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3464 return false;
3468 if (isSOPK(MI)) {
3469 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3470 if (Desc.isBranch()) {
3471 if (!Op->isMBB()) {
3472 ErrInfo = "invalid branch target for SOPK instruction";
3473 return false;
3475 } else {
3476 uint64_t Imm = Op->getImm();
3477 if (sopkIsZext(MI)) {
3478 if (!isUInt<16>(Imm)) {
3479 ErrInfo = "invalid immediate for SOPK instruction";
3480 return false;
3482 } else {
3483 if (!isInt<16>(Imm)) {
3484 ErrInfo = "invalid immediate for SOPK instruction";
3485 return false;
3491 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3492 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3493 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3494 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3495 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3496 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3498 const unsigned StaticNumOps = Desc.getNumOperands() +
3499 Desc.getNumImplicitUses();
3500 const unsigned NumImplicitOps = IsDst ? 2 : 1;
3502 // Allow additional implicit operands. This allows a fixup done by the post
3503 // RA scheduler where the main implicit operand is killed and implicit-defs
3504 // are added for sub-registers that remain live after this instruction.
3505 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3506 ErrInfo = "missing implicit register operands";
3507 return false;
3510 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3511 if (IsDst) {
3512 if (!Dst->isUse()) {
3513 ErrInfo = "v_movreld_b32 vdst should be a use operand";
3514 return false;
3517 unsigned UseOpIdx;
3518 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3519 UseOpIdx != StaticNumOps + 1) {
3520 ErrInfo = "movrel implicit operands should be tied";
3521 return false;
3525 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3526 const MachineOperand &ImpUse
3527 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3528 if (!ImpUse.isReg() || !ImpUse.isUse() ||
3529 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3530 ErrInfo = "src0 should be subreg of implicit vector use";
3531 return false;
3535 // Make sure we aren't losing exec uses in the td files. This mostly requires
3536 // being careful when using let Uses to try to add other use registers.
3537 if (shouldReadExec(MI)) {
3538 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3539 ErrInfo = "VALU instruction does not implicitly read exec mask";
3540 return false;
3544 if (isSMRD(MI)) {
3545 if (MI.mayStore()) {
3546 // The register offset form of scalar stores may only use m0 as the
3547 // soffset register.
3548 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3549 if (Soff && Soff->getReg() != AMDGPU::M0) {
3550 ErrInfo = "scalar stores must use m0 as offset register";
3551 return false;
3556 if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3557 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3558 if (Offset->getImm() != 0) {
3559 ErrInfo = "subtarget does not support offsets in flat instructions";
3560 return false;
3564 if (isMIMG(MI)) {
3565 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3566 if (DimOp) {
3567 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3568 AMDGPU::OpName::vaddr0);
3569 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3570 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3571 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3572 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
3573 const AMDGPU::MIMGDimInfo *Dim =
3574 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());
3576 if (!Dim) {
3577 ErrInfo = "dim is out of range";
3578 return false;
3581 bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3582 unsigned AddrWords = BaseOpcode->NumExtraArgs +
3583 (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3584 (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3585 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3587 unsigned VAddrWords;
3588 if (IsNSA) {
3589 VAddrWords = SRsrcIdx - VAddr0Idx;
3590 } else {
3591 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3592 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3593 if (AddrWords > 8)
3594 AddrWords = 16;
3595 else if (AddrWords > 4)
3596 AddrWords = 8;
3597 else if (AddrWords == 3 && VAddrWords == 4) {
3598 // CodeGen uses the V4 variant of instructions for three addresses,
3599 // because the selection DAG does not support non-power-of-two types.
3600 AddrWords = 4;
3604 if (VAddrWords != AddrWords) {
3605 ErrInfo = "bad vaddr size";
3606 return false;
3611 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3612 if (DppCt) {
3613 using namespace AMDGPU::DPP;
3615 unsigned DC = DppCt->getImm();
3616 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3617 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3618 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
3619 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
3620 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
3621 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
3622 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
3623 ErrInfo = "Invalid dpp_ctrl value";
3624 return false;
3626 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3627 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3628 ErrInfo = "Invalid dpp_ctrl value: "
3629 "wavefront shifts are not supported on GFX10+";
3630 return false;
3632 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3633 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3634 ErrInfo = "Invalid dpp_ctrl value: "
3635 "broadcasts are not supported on GFX10+";
3636 return false;
3638 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
3639 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3640 ErrInfo = "Invalid dpp_ctrl value: "
3641 "row_share and row_xmask are not supported before GFX10";
3642 return false;
3646 return true;
3649 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3650 switch (MI.getOpcode()) {
3651 default: return AMDGPU::INSTRUCTION_LIST_END;
3652 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3653 case AMDGPU::COPY: return AMDGPU::COPY;
3654 case AMDGPU::PHI: return AMDGPU::PHI;
3655 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3656 case AMDGPU::WQM: return AMDGPU::WQM;
3657 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
3658 case AMDGPU::WWM: return AMDGPU::WWM;
3659 case AMDGPU::S_MOV_B32: {
3660 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3661 return MI.getOperand(1).isReg() ||
3662 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3663 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3665 case AMDGPU::S_ADD_I32:
3666 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3667 case AMDGPU::S_ADDC_U32:
3668 return AMDGPU::V_ADDC_U32_e32;
3669 case AMDGPU::S_SUB_I32:
3670 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3671 // FIXME: These are not consistently handled, and selected when the carry is
3672 // used.
3673 case AMDGPU::S_ADD_U32:
3674 return AMDGPU::V_ADD_I32_e32;
3675 case AMDGPU::S_SUB_U32:
3676 return AMDGPU::V_SUB_I32_e32;
3677 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3678 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3679 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3680 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3681 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3682 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3683 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3684 case AMDGPU::S_XNOR_B32:
3685 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3686 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3687 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3688 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3689 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3690 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3691 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3692 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3693 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3694 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3695 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3696 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3697 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3698 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3699 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3700 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3701 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3702 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3703 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3704 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3705 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3706 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3707 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3708 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3709 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3710 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3711 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3712 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3713 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3714 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3715 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3716 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3717 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3718 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3719 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3720 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3721 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3722 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3723 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3725 llvm_unreachable(
3726 "Unexpected scalar opcode without corresponding vector one!");
3729 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
3730 unsigned OpNo) const {
3731 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3732 const MCInstrDesc &Desc = get(MI.getOpcode());
3733 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3734 Desc.OpInfo[OpNo].RegClass == -1) {
3735 Register Reg = MI.getOperand(OpNo).getReg();
3737 if (Register::isVirtualRegister(Reg))
3738 return MRI.getRegClass(Reg);
3739 return RI.getPhysRegClass(Reg);
3742 unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3743 return RI.getRegClass(RCID);
3746 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3747 MachineBasicBlock::iterator I = MI;
3748 MachineBasicBlock *MBB = MI.getParent();
3749 MachineOperand &MO = MI.getOperand(OpIdx);
3750 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
3751 const SIRegisterInfo *TRI =
3752 static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3753 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3754 const TargetRegisterClass *RC = RI.getRegClass(RCID);
3755 unsigned Size = TRI->getRegSizeInBits(*RC);
3756 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3757 if (MO.isReg())
3758 Opcode = AMDGPU::COPY;
3759 else if (RI.isSGPRClass(RC))
3760 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3762 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3763 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3764 VRC = &AMDGPU::VReg_64RegClass;
3765 else
3766 VRC = &AMDGPU::VGPR_32RegClass;
3768 Register Reg = MRI.createVirtualRegister(VRC);
3769 DebugLoc DL = MBB->findDebugLoc(I);
3770 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3771 MO.ChangeToRegister(Reg, false);
3774 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
3775 MachineRegisterInfo &MRI,
3776 MachineOperand &SuperReg,
3777 const TargetRegisterClass *SuperRC,
3778 unsigned SubIdx,
3779 const TargetRegisterClass *SubRC)
3780 const {
3781 MachineBasicBlock *MBB = MI->getParent();
3782 DebugLoc DL = MI->getDebugLoc();
3783 Register SubReg = MRI.createVirtualRegister(SubRC);
3785 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3786 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3787 .addReg(SuperReg.getReg(), 0, SubIdx);
3788 return SubReg;
3791 // Just in case the super register is itself a sub-register, copy it to a new
3792 // value so we don't need to worry about merging its subreg index with the
3793 // SubIdx passed to this function. The register coalescer should be able to
3794 // eliminate this extra copy.
3795 Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
3797 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3798 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3800 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3801 .addReg(NewSuperReg, 0, SubIdx);
3803 return SubReg;
3806 MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
3807 MachineBasicBlock::iterator MII,
3808 MachineRegisterInfo &MRI,
3809 MachineOperand &Op,
3810 const TargetRegisterClass *SuperRC,
3811 unsigned SubIdx,
3812 const TargetRegisterClass *SubRC) const {
3813 if (Op.isImm()) {
3814 if (SubIdx == AMDGPU::sub0)
3815 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3816 if (SubIdx == AMDGPU::sub1)
3817 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3819 llvm_unreachable("Unhandled register index for immediate");
3822 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3823 SubIdx, SubRC);
3824 return MachineOperand::CreateReg(SubReg, false);
3827 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3828 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3829 assert(Inst.getNumExplicitOperands() == 3);
3830 MachineOperand Op1 = Inst.getOperand(1);
3831 Inst.RemoveOperand(1);
3832 Inst.addOperand(Op1);
3835 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
3836 const MCOperandInfo &OpInfo,
3837 const MachineOperand &MO) const {
3838 if (!MO.isReg())
3839 return false;
3841 Register Reg = MO.getReg();
3842 const TargetRegisterClass *RC = Register::isVirtualRegister(Reg)
3843 ? MRI.getRegClass(Reg)
3844 : RI.getPhysRegClass(Reg);
3846 const SIRegisterInfo *TRI =
3847 static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3848 RC = TRI->getSubRegClass(RC, MO.getSubReg());
3850 // In order to be legal, the common sub-class must be equal to the
3851 // class of the current operand. For example:
3853 // v_mov_b32 s0 ; Operand defined as vsrc_b32
3854 // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3856 // s_sendmsg 0, s0 ; Operand defined as m0reg
3857 // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3859 return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3862 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
3863 const MCOperandInfo &OpInfo,
3864 const MachineOperand &MO) const {
3865 if (MO.isReg())
3866 return isLegalRegOperand(MRI, OpInfo, MO);
3868 // Handle non-register types that are treated like immediates.
3869 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3870 return true;
3873 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3874 const MachineOperand *MO) const {
3875 const MachineFunction &MF = *MI.getParent()->getParent();
3876 const MachineRegisterInfo &MRI = MF.getRegInfo();
3877 const MCInstrDesc &InstDesc = MI.getDesc();
3878 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3879 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3880 const TargetRegisterClass *DefinedRC =
3881 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3882 if (!MO)
3883 MO = &MI.getOperand(OpIdx);
3885 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3886 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3887 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3888 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3889 return false;
3891 SmallDenseSet<RegSubRegPair> SGPRsUsed;
3892 if (MO->isReg())
3893 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3895 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3896 if (i == OpIdx)
3897 continue;
3898 const MachineOperand &Op = MI.getOperand(i);
3899 if (Op.isReg()) {
3900 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3901 if (!SGPRsUsed.count(SGPR) &&
3902 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3903 if (--ConstantBusLimit <= 0)
3904 return false;
3905 SGPRsUsed.insert(SGPR);
3907 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3908 if (--ConstantBusLimit <= 0)
3909 return false;
3910 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3911 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3912 if (!VOP3LiteralLimit--)
3913 return false;
3914 if (--ConstantBusLimit <= 0)
3915 return false;
3920 if (MO->isReg()) {
3921 assert(DefinedRC);
3922 return isLegalRegOperand(MRI, OpInfo, *MO);
3925 // Handle non-register types that are treated like immediates.
3926 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
3928 if (!DefinedRC) {
3929 // This operand expects an immediate.
3930 return true;
3933 return isImmOperandLegal(MI, OpIdx, *MO);
3936 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
3937 MachineInstr &MI) const {
3938 unsigned Opc = MI.getOpcode();
3939 const MCInstrDesc &InstrDesc = get(Opc);
3941 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3942 MachineOperand &Src0 = MI.getOperand(Src0Idx);
3944 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3945 MachineOperand &Src1 = MI.getOperand(Src1Idx);
3947 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3948 // we need to only have one constant bus use before GFX10.
3949 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3950 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
3951 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
3952 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
3953 legalizeOpWithMove(MI, Src0Idx);
3955 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3956 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3957 // src0/src1 with V_READFIRSTLANE.
3958 if (Opc == AMDGPU::V_WRITELANE_B32) {
3959 const DebugLoc &DL = MI.getDebugLoc();
3960 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3961 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3962 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3963 .add(Src0);
3964 Src0.ChangeToRegister(Reg, false);
3966 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3967 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3968 const DebugLoc &DL = MI.getDebugLoc();
3969 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3970 .add(Src1);
3971 Src1.ChangeToRegister(Reg, false);
3973 return;
3976 // No VOP2 instructions support AGPRs.
3977 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
3978 legalizeOpWithMove(MI, Src0Idx);
3980 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
3981 legalizeOpWithMove(MI, Src1Idx);
3983 // VOP2 src0 instructions support all operand types, so we don't need to check
3984 // their legality. If src1 is already legal, we don't need to do anything.
3985 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3986 return;
3988 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3989 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3990 // select is uniform.
3991 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3992 RI.isVGPR(MRI, Src1.getReg())) {
3993 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3994 const DebugLoc &DL = MI.getDebugLoc();
3995 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3996 .add(Src1);
3997 Src1.ChangeToRegister(Reg, false);
3998 return;
4001 // We do not use commuteInstruction here because it is too aggressive and will
4002 // commute if it is possible. We only want to commute here if it improves
4003 // legality. This can be called a fairly large number of times so don't waste
4004 // compile time pointlessly swapping and checking legality again.
4005 if (HasImplicitSGPR || !MI.isCommutable()) {
4006 legalizeOpWithMove(MI, Src1Idx);
4007 return;
4010 // If src0 can be used as src1, commuting will make the operands legal.
4011 // Otherwise we have to give up and insert a move.
4013 // TODO: Other immediate-like operand kinds could be commuted if there was a
4014 // MachineOperand::ChangeTo* for them.
4015 if ((!Src1.isImm() && !Src1.isReg()) ||
4016 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4017 legalizeOpWithMove(MI, Src1Idx);
4018 return;
4021 int CommutedOpc = commuteOpcode(MI);
4022 if (CommutedOpc == -1) {
4023 legalizeOpWithMove(MI, Src1Idx);
4024 return;
4027 MI.setDesc(get(CommutedOpc));
4029 Register Src0Reg = Src0.getReg();
4030 unsigned Src0SubReg = Src0.getSubReg();
4031 bool Src0Kill = Src0.isKill();
4033 if (Src1.isImm())
4034 Src0.ChangeToImmediate(Src1.getImm());
4035 else if (Src1.isReg()) {
4036 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4037 Src0.setSubReg(Src1.getSubReg());
4038 } else
4039 llvm_unreachable("Should only have register or immediate operands");
4041 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4042 Src1.setSubReg(Src0SubReg);
4043 fixImplicitOperands(MI);
4046 // Legalize VOP3 operands. All operand types are supported for any operand
4047 // but only one literal constant and only starting from GFX10.
4048 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
4049 MachineInstr &MI) const {
4050 unsigned Opc = MI.getOpcode();
4052 int VOP3Idx[3] = {
4053 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4054 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4055 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4058 if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4059 Opc == AMDGPU::V_PERMLANEX16_B32) {
4060 // src1 and src2 must be scalar
4061 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4062 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4063 const DebugLoc &DL = MI.getDebugLoc();
4064 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4065 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4066 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4067 .add(Src1);
4068 Src1.ChangeToRegister(Reg, false);
4070 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4071 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4072 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4073 .add(Src2);
4074 Src2.ChangeToRegister(Reg, false);
4078 // Find the one SGPR operand we are allowed to use.
4079 int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4080 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4081 SmallDenseSet<unsigned> SGPRsUsed;
4082 unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4083 if (SGPRReg != AMDGPU::NoRegister) {
4084 SGPRsUsed.insert(SGPRReg);
4085 --ConstantBusLimit;
4088 for (unsigned i = 0; i < 3; ++i) {
4089 int Idx = VOP3Idx[i];
4090 if (Idx == -1)
4091 break;
4092 MachineOperand &MO = MI.getOperand(Idx);
4094 if (!MO.isReg()) {
4095 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4096 continue;
4098 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4099 --LiteralLimit;
4100 --ConstantBusLimit;
4101 continue;
4104 --LiteralLimit;
4105 --ConstantBusLimit;
4106 legalizeOpWithMove(MI, Idx);
4107 continue;
4110 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4111 !isOperandLegal(MI, Idx, &MO)) {
4112 legalizeOpWithMove(MI, Idx);
4113 continue;
4116 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4117 continue; // VGPRs are legal
4119 // We can use one SGPR in each VOP3 instruction prior to GFX10
4120 // and two starting from GFX10.
4121 if (SGPRsUsed.count(MO.getReg()))
4122 continue;
4123 if (ConstantBusLimit > 0) {
4124 SGPRsUsed.insert(MO.getReg());
4125 --ConstantBusLimit;
4126 continue;
4129 // If we make it this far, then the operand is not legal and we must
4130 // legalize it.
4131 legalizeOpWithMove(MI, Idx);
4135 unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
4136 MachineRegisterInfo &MRI) const {
4137 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4138 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4139 Register DstReg = MRI.createVirtualRegister(SRC);
4140 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4142 if (RI.hasAGPRs(VRC)) {
4143 VRC = RI.getEquivalentVGPRClass(VRC);
4144 Register NewSrcReg = MRI.createVirtualRegister(VRC);
4145 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4146 get(TargetOpcode::COPY), NewSrcReg)
4147 .addReg(SrcReg);
4148 SrcReg = NewSrcReg;
4151 if (SubRegs == 1) {
4152 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4153 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4154 .addReg(SrcReg);
4155 return DstReg;
4158 SmallVector<unsigned, 8> SRegs;
4159 for (unsigned i = 0; i < SubRegs; ++i) {
4160 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4161 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4162 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4163 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4164 SRegs.push_back(SGPR);
4167 MachineInstrBuilder MIB =
4168 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4169 get(AMDGPU::REG_SEQUENCE), DstReg);
4170 for (unsigned i = 0; i < SubRegs; ++i) {
4171 MIB.addReg(SRegs[i]);
4172 MIB.addImm(RI.getSubRegFromChannel(i));
4174 return DstReg;
4177 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
4178 MachineInstr &MI) const {
4180 // If the pointer is store in VGPRs, then we need to move them to
4181 // SGPRs using v_readfirstlane. This is safe because we only select
4182 // loads with uniform pointers to SMRD instruction so we know the
4183 // pointer value is uniform.
4184 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4185 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4186 unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4187 SBase->setReg(SGPR);
4189 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4190 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4191 unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4192 SOff->setReg(SGPR);
4196 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
4197 MachineBasicBlock::iterator I,
4198 const TargetRegisterClass *DstRC,
4199 MachineOperand &Op,
4200 MachineRegisterInfo &MRI,
4201 const DebugLoc &DL) const {
4202 Register OpReg = Op.getReg();
4203 unsigned OpSubReg = Op.getSubReg();
4205 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4206 RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4208 // Check if operand is already the correct register class.
4209 if (DstRC == OpRC)
4210 return;
4212 Register DstReg = MRI.createVirtualRegister(DstRC);
4213 MachineInstr *Copy =
4214 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4216 Op.setReg(DstReg);
4217 Op.setSubReg(0);
4219 MachineInstr *Def = MRI.getVRegDef(OpReg);
4220 if (!Def)
4221 return;
4223 // Try to eliminate the copy if it is copying an immediate value.
4224 if (Def->isMoveImmediate())
4225 FoldImmediate(*Copy, *Def, OpReg, &MRI);
4227 bool ImpDef = Def->isImplicitDef();
4228 while (!ImpDef && Def && Def->isCopy()) {
4229 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4230 ImpDef = Def && Def->isImplicitDef();
4232 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
4233 !ImpDef)
4234 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4237 // Emit the actual waterfall loop, executing the wrapped instruction for each
4238 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4239 // iteration, in the worst case we execute 64 (once per lane).
4240 static void
4241 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
4242 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4243 const DebugLoc &DL, MachineOperand &Rsrc) {
4244 MachineFunction &MF = *OrigBB.getParent();
4245 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4246 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4247 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4248 unsigned SaveExecOpc =
4249 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4250 unsigned XorTermOpc =
4251 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4252 unsigned AndOpc =
4253 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4254 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4256 MachineBasicBlock::iterator I = LoopBB.begin();
4258 Register VRsrc = Rsrc.getReg();
4259 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4261 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4262 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4263 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4264 Register AndCond = MRI.createVirtualRegister(BoolXExecRC);
4265 Register SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4266 Register SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4267 Register SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4268 Register SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4269 Register SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4271 // Beginning of the loop, read the next Rsrc variant.
4272 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4273 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4274 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4275 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4276 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4277 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4278 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4279 .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4281 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4282 .addReg(SRsrcSub0)
4283 .addImm(AMDGPU::sub0)
4284 .addReg(SRsrcSub1)
4285 .addImm(AMDGPU::sub1)
4286 .addReg(SRsrcSub2)
4287 .addImm(AMDGPU::sub2)
4288 .addReg(SRsrcSub3)
4289 .addImm(AMDGPU::sub3);
4291 // Update Rsrc operand to use the SGPR Rsrc.
4292 Rsrc.setReg(SRsrc);
4293 Rsrc.setIsKill(true);
4295 // Identify all lanes with identical Rsrc operands in their VGPRs.
4296 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4297 .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4298 .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4299 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4300 .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4301 .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4302 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4303 .addReg(CondReg0)
4304 .addReg(CondReg1);
4306 MRI.setSimpleHint(SaveExec, AndCond);
4308 // Update EXEC to matching lanes, saving original to SaveExec.
4309 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4310 .addReg(AndCond, RegState::Kill);
4312 // The original instruction is here; we insert the terminators after it.
4313 I = LoopBB.end();
4315 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4316 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4317 .addReg(Exec)
4318 .addReg(SaveExec);
4319 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4322 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4323 // with SGPRs by iterating over all unique values across all lanes.
4324 static void loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
4325 MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4326 MachineBasicBlock &MBB = *MI.getParent();
4327 MachineFunction &MF = *MBB.getParent();
4328 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4329 const SIRegisterInfo *TRI = ST.getRegisterInfo();
4330 MachineRegisterInfo &MRI = MF.getRegInfo();
4331 MachineBasicBlock::iterator I(&MI);
4332 const DebugLoc &DL = MI.getDebugLoc();
4333 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4334 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4335 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4337 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4339 // Save the EXEC mask
4340 BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4342 // Killed uses in the instruction we are waterfalling around will be
4343 // incorrect due to the added control-flow.
4344 for (auto &MO : MI.uses()) {
4345 if (MO.isReg() && MO.isUse()) {
4346 MRI.clearKillFlags(MO.getReg());
4350 // To insert the loop we need to split the block. Move everything after this
4351 // point to a new block, and insert a new empty block between the two.
4352 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
4353 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4354 MachineFunction::iterator MBBI(MBB);
4355 ++MBBI;
4357 MF.insert(MBBI, LoopBB);
4358 MF.insert(MBBI, RemainderBB);
4360 LoopBB->addSuccessor(LoopBB);
4361 LoopBB->addSuccessor(RemainderBB);
4363 // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4364 MachineBasicBlock::iterator J = I++;
4365 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4366 RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4367 LoopBB->splice(LoopBB->begin(), &MBB, J);
4369 MBB.addSuccessor(LoopBB);
4371 // Update dominators. We know that MBB immediately dominates LoopBB, that
4372 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4373 // dominates all of the successors transferred to it from MBB that MBB used
4374 // to dominate.
4375 if (MDT) {
4376 MDT->addNewBlock(LoopBB, &MBB);
4377 MDT->addNewBlock(RemainderBB, LoopBB);
4378 for (auto &Succ : RemainderBB->successors()) {
4379 if (MDT->dominates(&MBB, Succ)) {
4380 MDT->changeImmediateDominator(Succ, RemainderBB);
4385 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4387 // Restore the EXEC mask
4388 MachineBasicBlock::iterator First = RemainderBB->begin();
4389 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4392 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4393 static std::tuple<unsigned, unsigned>
4394 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
4395 MachineBasicBlock &MBB = *MI.getParent();
4396 MachineFunction &MF = *MBB.getParent();
4397 MachineRegisterInfo &MRI = MF.getRegInfo();
4399 // Extract the ptr from the resource descriptor.
4400 unsigned RsrcPtr =
4401 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4402 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4404 // Create an empty resource descriptor
4405 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4406 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4407 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4408 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4409 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4411 // Zero64 = 0
4412 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4413 .addImm(0);
4415 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4416 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4417 .addImm(RsrcDataFormat & 0xFFFFFFFF);
4419 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4420 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4421 .addImm(RsrcDataFormat >> 32);
4423 // NewSRsrc = {Zero64, SRsrcFormat}
4424 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4425 .addReg(Zero64)
4426 .addImm(AMDGPU::sub0_sub1)
4427 .addReg(SRsrcFormatLo)
4428 .addImm(AMDGPU::sub2)
4429 .addReg(SRsrcFormatHi)
4430 .addImm(AMDGPU::sub3);
4432 return std::make_tuple(RsrcPtr, NewSRsrc);
4435 void SIInstrInfo::legalizeOperands(MachineInstr &MI,
4436 MachineDominatorTree *MDT) const {
4437 MachineFunction &MF = *MI.getParent()->getParent();
4438 MachineRegisterInfo &MRI = MF.getRegInfo();
4440 // Legalize VOP2
4441 if (isVOP2(MI) || isVOPC(MI)) {
4442 legalizeOperandsVOP2(MRI, MI);
4443 return;
4446 // Legalize VOP3
4447 if (isVOP3(MI)) {
4448 legalizeOperandsVOP3(MRI, MI);
4449 return;
4452 // Legalize SMRD
4453 if (isSMRD(MI)) {
4454 legalizeOperandsSMRD(MRI, MI);
4455 return;
4458 // Legalize REG_SEQUENCE and PHI
4459 // The register class of the operands much be the same type as the register
4460 // class of the output.
4461 if (MI.getOpcode() == AMDGPU::PHI) {
4462 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4463 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4464 if (!MI.getOperand(i).isReg() ||
4465 !Register::isVirtualRegister(MI.getOperand(i).getReg()))
4466 continue;
4467 const TargetRegisterClass *OpRC =
4468 MRI.getRegClass(MI.getOperand(i).getReg());
4469 if (RI.hasVectorRegisters(OpRC)) {
4470 VRC = OpRC;
4471 } else {
4472 SRC = OpRC;
4476 // If any of the operands are VGPR registers, then they all most be
4477 // otherwise we will create illegal VGPR->SGPR copies when legalizing
4478 // them.
4479 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4480 if (!VRC) {
4481 assert(SRC);
4482 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) ? RI.getEquivalentAGPRClass(SRC)
4483 : RI.getEquivalentVGPRClass(SRC);
4485 RC = VRC;
4486 } else {
4487 RC = SRC;
4490 // Update all the operands so they have the same type.
4491 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4492 MachineOperand &Op = MI.getOperand(I);
4493 if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4494 continue;
4496 // MI is a PHI instruction.
4497 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4498 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4500 // Avoid creating no-op copies with the same src and dst reg class. These
4501 // confuse some of the machine passes.
4502 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4506 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4507 // VGPR dest type and SGPR sources, insert copies so all operands are
4508 // VGPRs. This seems to help operand folding / the register coalescer.
4509 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4510 MachineBasicBlock *MBB = MI.getParent();
4511 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4512 if (RI.hasVGPRs(DstRC)) {
4513 // Update all the operands so they are VGPR register classes. These may
4514 // not be the same register class because REG_SEQUENCE supports mixing
4515 // subregister index types e.g. sub0_sub1 + sub2 + sub3
4516 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4517 MachineOperand &Op = MI.getOperand(I);
4518 if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4519 continue;
4521 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4522 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4523 if (VRC == OpRC)
4524 continue;
4526 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4527 Op.setIsKill();
4531 return;
4534 // Legalize INSERT_SUBREG
4535 // src0 must have the same register class as dst
4536 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4537 Register Dst = MI.getOperand(0).getReg();
4538 Register Src0 = MI.getOperand(1).getReg();
4539 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4540 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4541 if (DstRC != Src0RC) {
4542 MachineBasicBlock *MBB = MI.getParent();
4543 MachineOperand &Op = MI.getOperand(1);
4544 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4546 return;
4549 // Legalize SI_INIT_M0
4550 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4551 MachineOperand &Src = MI.getOperand(0);
4552 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4553 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4554 return;
4557 // Legalize MIMG and MUBUF/MTBUF for shaders.
4559 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4560 // scratch memory access. In both cases, the legalization never involves
4561 // conversion to the addr64 form.
4562 if (isMIMG(MI) ||
4563 (AMDGPU::isShader(MF.getFunction().getCallingConv()) &&
4564 (isMUBUF(MI) || isMTBUF(MI)))) {
4565 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4566 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4567 unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4568 SRsrc->setReg(SGPR);
4571 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4572 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4573 unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4574 SSamp->setReg(SGPR);
4576 return;
4579 // Legalize MUBUF* instructions.
4580 int RsrcIdx =
4581 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4582 if (RsrcIdx != -1) {
4583 // We have an MUBUF instruction
4584 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4585 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4586 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4587 RI.getRegClass(RsrcRC))) {
4588 // The operands are legal.
4589 // FIXME: We may need to legalize operands besided srsrc.
4590 return;
4593 // Legalize a VGPR Rsrc.
4595 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4596 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4597 // a zero-value SRsrc.
4599 // If the instruction is _OFFSET (both idxen and offen disabled), and we
4600 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4601 // above.
4603 // Otherwise we are on non-ADDR64 hardware, and/or we have
4604 // idxen/offen/bothen and we fall back to a waterfall loop.
4606 MachineBasicBlock &MBB = *MI.getParent();
4608 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4609 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4610 // This is already an ADDR64 instruction so we need to add the pointer
4611 // extracted from the resource descriptor to the current value of VAddr.
4612 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4613 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4614 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4616 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4617 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4618 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4620 unsigned RsrcPtr, NewSRsrc;
4621 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4623 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4624 const DebugLoc &DL = MI.getDebugLoc();
4625 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4626 .addDef(CondReg0)
4627 .addReg(RsrcPtr, 0, AMDGPU::sub0)
4628 .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4629 .addImm(0);
4631 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4632 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4633 .addDef(CondReg1, RegState::Dead)
4634 .addReg(RsrcPtr, 0, AMDGPU::sub1)
4635 .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4636 .addReg(CondReg0, RegState::Kill)
4637 .addImm(0);
4639 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4640 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4641 .addReg(NewVAddrLo)
4642 .addImm(AMDGPU::sub0)
4643 .addReg(NewVAddrHi)
4644 .addImm(AMDGPU::sub1);
4646 VAddr->setReg(NewVAddr);
4647 Rsrc->setReg(NewSRsrc);
4648 } else if (!VAddr && ST.hasAddr64()) {
4649 // This instructions is the _OFFSET variant, so we need to convert it to
4650 // ADDR64.
4651 assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4652 < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
4653 "FIXME: Need to emit flat atomics here");
4655 unsigned RsrcPtr, NewSRsrc;
4656 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4658 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4659 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4660 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4661 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4662 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4664 // Atomics rith return have have an additional tied operand and are
4665 // missing some of the special bits.
4666 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4667 MachineInstr *Addr64;
4669 if (!VDataIn) {
4670 // Regular buffer load / store.
4671 MachineInstrBuilder MIB =
4672 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4673 .add(*VData)
4674 .addReg(NewVAddr)
4675 .addReg(NewSRsrc)
4676 .add(*SOffset)
4677 .add(*Offset);
4679 // Atomics do not have this operand.
4680 if (const MachineOperand *GLC =
4681 getNamedOperand(MI, AMDGPU::OpName::glc)) {
4682 MIB.addImm(GLC->getImm());
4684 if (const MachineOperand *DLC =
4685 getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4686 MIB.addImm(DLC->getImm());
4689 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4691 if (const MachineOperand *TFE =
4692 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4693 MIB.addImm(TFE->getImm());
4696 MIB.cloneMemRefs(MI);
4697 Addr64 = MIB;
4698 } else {
4699 // Atomics with return.
4700 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4701 .add(*VData)
4702 .add(*VDataIn)
4703 .addReg(NewVAddr)
4704 .addReg(NewSRsrc)
4705 .add(*SOffset)
4706 .add(*Offset)
4707 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4708 .cloneMemRefs(MI);
4711 MI.removeFromParent();
4713 // NewVaddr = {NewVaddrHi, NewVaddrLo}
4714 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4715 NewVAddr)
4716 .addReg(RsrcPtr, 0, AMDGPU::sub0)
4717 .addImm(AMDGPU::sub0)
4718 .addReg(RsrcPtr, 0, AMDGPU::sub1)
4719 .addImm(AMDGPU::sub1);
4720 } else {
4721 // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4722 // to SGPRs.
4723 loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4728 void SIInstrInfo::moveToVALU(MachineInstr &TopInst,
4729 MachineDominatorTree *MDT) const {
4730 SetVectorType Worklist;
4731 Worklist.insert(&TopInst);
4733 while (!Worklist.empty()) {
4734 MachineInstr &Inst = *Worklist.pop_back_val();
4735 MachineBasicBlock *MBB = Inst.getParent();
4736 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
4738 unsigned Opcode = Inst.getOpcode();
4739 unsigned NewOpcode = getVALUOp(Inst);
4741 // Handle some special cases
4742 switch (Opcode) {
4743 default:
4744 break;
4745 case AMDGPU::S_ADD_U64_PSEUDO:
4746 case AMDGPU::S_SUB_U64_PSEUDO:
4747 splitScalar64BitAddSub(Worklist, Inst, MDT);
4748 Inst.eraseFromParent();
4749 continue;
4750 case AMDGPU::S_ADD_I32:
4751 case AMDGPU::S_SUB_I32:
4752 // FIXME: The u32 versions currently selected use the carry.
4753 if (moveScalarAddSub(Worklist, Inst, MDT))
4754 continue;
4756 // Default handling
4757 break;
4758 case AMDGPU::S_AND_B64:
4759 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4760 Inst.eraseFromParent();
4761 continue;
4763 case AMDGPU::S_OR_B64:
4764 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4765 Inst.eraseFromParent();
4766 continue;
4768 case AMDGPU::S_XOR_B64:
4769 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4770 Inst.eraseFromParent();
4771 continue;
4773 case AMDGPU::S_NAND_B64:
4774 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4775 Inst.eraseFromParent();
4776 continue;
4778 case AMDGPU::S_NOR_B64:
4779 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4780 Inst.eraseFromParent();
4781 continue;
4783 case AMDGPU::S_XNOR_B64:
4784 if (ST.hasDLInsts())
4785 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4786 else
4787 splitScalar64BitXnor(Worklist, Inst, MDT);
4788 Inst.eraseFromParent();
4789 continue;
4791 case AMDGPU::S_ANDN2_B64:
4792 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4793 Inst.eraseFromParent();
4794 continue;
4796 case AMDGPU::S_ORN2_B64:
4797 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4798 Inst.eraseFromParent();
4799 continue;
4801 case AMDGPU::S_NOT_B64:
4802 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4803 Inst.eraseFromParent();
4804 continue;
4806 case AMDGPU::S_BCNT1_I32_B64:
4807 splitScalar64BitBCNT(Worklist, Inst);
4808 Inst.eraseFromParent();
4809 continue;
4811 case AMDGPU::S_BFE_I64:
4812 splitScalar64BitBFE(Worklist, Inst);
4813 Inst.eraseFromParent();
4814 continue;
4816 case AMDGPU::S_LSHL_B32:
4817 if (ST.hasOnlyRevVALUShifts()) {
4818 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4819 swapOperands(Inst);
4821 break;
4822 case AMDGPU::S_ASHR_I32:
4823 if (ST.hasOnlyRevVALUShifts()) {
4824 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4825 swapOperands(Inst);
4827 break;
4828 case AMDGPU::S_LSHR_B32:
4829 if (ST.hasOnlyRevVALUShifts()) {
4830 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4831 swapOperands(Inst);
4833 break;
4834 case AMDGPU::S_LSHL_B64:
4835 if (ST.hasOnlyRevVALUShifts()) {
4836 NewOpcode = AMDGPU::V_LSHLREV_B64;
4837 swapOperands(Inst);
4839 break;
4840 case AMDGPU::S_ASHR_I64:
4841 if (ST.hasOnlyRevVALUShifts()) {
4842 NewOpcode = AMDGPU::V_ASHRREV_I64;
4843 swapOperands(Inst);
4845 break;
4846 case AMDGPU::S_LSHR_B64:
4847 if (ST.hasOnlyRevVALUShifts()) {
4848 NewOpcode = AMDGPU::V_LSHRREV_B64;
4849 swapOperands(Inst);
4851 break;
4853 case AMDGPU::S_ABS_I32:
4854 lowerScalarAbs(Worklist, Inst);
4855 Inst.eraseFromParent();
4856 continue;
4858 case AMDGPU::S_CBRANCH_SCC0:
4859 case AMDGPU::S_CBRANCH_SCC1:
4860 // Clear unused bits of vcc
4861 if (ST.isWave32())
4862 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4863 AMDGPU::VCC_LO)
4864 .addReg(AMDGPU::EXEC_LO)
4865 .addReg(AMDGPU::VCC_LO);
4866 else
4867 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4868 AMDGPU::VCC)
4869 .addReg(AMDGPU::EXEC)
4870 .addReg(AMDGPU::VCC);
4871 break;
4873 case AMDGPU::S_BFE_U64:
4874 case AMDGPU::S_BFM_B64:
4875 llvm_unreachable("Moving this op to VALU not implemented");
4877 case AMDGPU::S_PACK_LL_B32_B16:
4878 case AMDGPU::S_PACK_LH_B32_B16:
4879 case AMDGPU::S_PACK_HH_B32_B16:
4880 movePackToVALU(Worklist, MRI, Inst);
4881 Inst.eraseFromParent();
4882 continue;
4884 case AMDGPU::S_XNOR_B32:
4885 lowerScalarXnor(Worklist, Inst);
4886 Inst.eraseFromParent();
4887 continue;
4889 case AMDGPU::S_NAND_B32:
4890 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4891 Inst.eraseFromParent();
4892 continue;
4894 case AMDGPU::S_NOR_B32:
4895 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4896 Inst.eraseFromParent();
4897 continue;
4899 case AMDGPU::S_ANDN2_B32:
4900 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4901 Inst.eraseFromParent();
4902 continue;
4904 case AMDGPU::S_ORN2_B32:
4905 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4906 Inst.eraseFromParent();
4907 continue;
4910 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4911 // We cannot move this instruction to the VALU, so we should try to
4912 // legalize its operands instead.
4913 legalizeOperands(Inst, MDT);
4914 continue;
4917 // Use the new VALU Opcode.
4918 const MCInstrDesc &NewDesc = get(NewOpcode);
4919 Inst.setDesc(NewDesc);
4921 // Remove any references to SCC. Vector instructions can't read from it, and
4922 // We're just about to add the implicit use / defs of VCC, and we don't want
4923 // both.
4924 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4925 MachineOperand &Op = Inst.getOperand(i);
4926 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4927 // Only propagate through live-def of SCC.
4928 if (Op.isDef() && !Op.isDead())
4929 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4930 Inst.RemoveOperand(i);
4934 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4935 // We are converting these to a BFE, so we need to add the missing
4936 // operands for the size and offset.
4937 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4938 Inst.addOperand(MachineOperand::CreateImm(0));
4939 Inst.addOperand(MachineOperand::CreateImm(Size));
4941 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4942 // The VALU version adds the second operand to the result, so insert an
4943 // extra 0 operand.
4944 Inst.addOperand(MachineOperand::CreateImm(0));
4947 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
4948 fixImplicitOperands(Inst);
4950 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4951 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4952 // If we need to move this to VGPRs, we need to unpack the second operand
4953 // back into the 2 separate ones for bit offset and width.
4954 assert(OffsetWidthOp.isImm() &&
4955 "Scalar BFE is only implemented for constant width and offset");
4956 uint32_t Imm = OffsetWidthOp.getImm();
4958 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4959 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4960 Inst.RemoveOperand(2); // Remove old immediate.
4961 Inst.addOperand(MachineOperand::CreateImm(Offset));
4962 Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4965 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4966 unsigned NewDstReg = AMDGPU::NoRegister;
4967 if (HasDst) {
4968 Register DstReg = Inst.getOperand(0).getReg();
4969 if (Register::isPhysicalRegister(DstReg))
4970 continue;
4972 // Update the destination register class.
4973 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4974 if (!NewDstRC)
4975 continue;
4977 if (Inst.isCopy() &&
4978 Register::isVirtualRegister(Inst.getOperand(1).getReg()) &&
4979 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4980 // Instead of creating a copy where src and dst are the same register
4981 // class, we just replace all uses of dst with src. These kinds of
4982 // copies interfere with the heuristics MachineSink uses to decide
4983 // whether or not to split a critical edge. Since the pass assumes
4984 // that copies will end up as machine instructions and not be
4985 // eliminated.
4986 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4987 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4988 MRI.clearKillFlags(Inst.getOperand(1).getReg());
4989 Inst.getOperand(0).setReg(DstReg);
4991 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4992 // these are deleted later, but at -O0 it would leave a suspicious
4993 // looking illegal copy of an undef register.
4994 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4995 Inst.RemoveOperand(I);
4996 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4997 continue;
5000 NewDstReg = MRI.createVirtualRegister(NewDstRC);
5001 MRI.replaceRegWith(DstReg, NewDstReg);
5004 // Legalize the operands
5005 legalizeOperands(Inst, MDT);
5007 if (HasDst)
5008 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
5012 // Add/sub require special handling to deal with carry outs.
5013 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
5014 MachineDominatorTree *MDT) const {
5015 if (ST.hasAddNoCarry()) {
5016 // Assume there is no user of scc since we don't select this in that case.
5017 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5018 // is used.
5020 MachineBasicBlock &MBB = *Inst.getParent();
5021 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5023 Register OldDstReg = Inst.getOperand(0).getReg();
5024 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5026 unsigned Opc = Inst.getOpcode();
5027 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
5029 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
5030 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
5032 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5033 Inst.RemoveOperand(3);
5035 Inst.setDesc(get(NewOpc));
5036 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5037 Inst.addImplicitDefUseOperands(*MBB.getParent());
5038 MRI.replaceRegWith(OldDstReg, ResultReg);
5039 legalizeOperands(Inst, MDT);
5041 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5042 return true;
5045 return false;
5048 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5049 MachineInstr &Inst) const {
5050 MachineBasicBlock &MBB = *Inst.getParent();
5051 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5052 MachineBasicBlock::iterator MII = Inst;
5053 DebugLoc DL = Inst.getDebugLoc();
5055 MachineOperand &Dest = Inst.getOperand(0);
5056 MachineOperand &Src = Inst.getOperand(1);
5057 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5058 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5060 unsigned SubOp = ST.hasAddNoCarry() ?
5061 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5063 BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5064 .addImm(0)
5065 .addReg(Src.getReg());
5067 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5068 .addReg(Src.getReg())
5069 .addReg(TmpReg);
5071 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5072 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5075 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5076 MachineInstr &Inst) const {
5077 MachineBasicBlock &MBB = *Inst.getParent();
5078 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5079 MachineBasicBlock::iterator MII = Inst;
5080 const DebugLoc &DL = Inst.getDebugLoc();
5082 MachineOperand &Dest = Inst.getOperand(0);
5083 MachineOperand &Src0 = Inst.getOperand(1);
5084 MachineOperand &Src1 = Inst.getOperand(2);
5086 if (ST.hasDLInsts()) {
5087 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5088 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5089 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5091 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5092 .add(Src0)
5093 .add(Src1);
5095 MRI.replaceRegWith(Dest.getReg(), NewDest);
5096 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5097 } else {
5098 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5099 // invert either source and then perform the XOR. If either source is a
5100 // scalar register, then we can leave the inversion on the scalar unit to
5101 // acheive a better distrubution of scalar and vector instructions.
5102 bool Src0IsSGPR = Src0.isReg() &&
5103 RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5104 bool Src1IsSGPR = Src1.isReg() &&
5105 RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5106 MachineInstr *Xor;
5107 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5108 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5110 // Build a pair of scalar instructions and add them to the work list.
5111 // The next iteration over the work list will lower these to the vector
5112 // unit as necessary.
5113 if (Src0IsSGPR) {
5114 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5115 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5116 .addReg(Temp)
5117 .add(Src1);
5118 } else if (Src1IsSGPR) {
5119 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5120 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5121 .add(Src0)
5122 .addReg(Temp);
5123 } else {
5124 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5125 .add(Src0)
5126 .add(Src1);
5127 MachineInstr *Not =
5128 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5129 Worklist.insert(Not);
5132 MRI.replaceRegWith(Dest.getReg(), NewDest);
5134 Worklist.insert(Xor);
5136 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5140 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5141 MachineInstr &Inst,
5142 unsigned Opcode) const {
5143 MachineBasicBlock &MBB = *Inst.getParent();
5144 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5145 MachineBasicBlock::iterator MII = Inst;
5146 const DebugLoc &DL = Inst.getDebugLoc();
5148 MachineOperand &Dest = Inst.getOperand(0);
5149 MachineOperand &Src0 = Inst.getOperand(1);
5150 MachineOperand &Src1 = Inst.getOperand(2);
5152 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5153 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5155 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5156 .add(Src0)
5157 .add(Src1);
5159 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5160 .addReg(Interm);
5162 Worklist.insert(&Op);
5163 Worklist.insert(&Not);
5165 MRI.replaceRegWith(Dest.getReg(), NewDest);
5166 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5169 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5170 MachineInstr &Inst,
5171 unsigned Opcode) const {
5172 MachineBasicBlock &MBB = *Inst.getParent();
5173 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5174 MachineBasicBlock::iterator MII = Inst;
5175 const DebugLoc &DL = Inst.getDebugLoc();
5177 MachineOperand &Dest = Inst.getOperand(0);
5178 MachineOperand &Src0 = Inst.getOperand(1);
5179 MachineOperand &Src1 = Inst.getOperand(2);
5181 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5182 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5184 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5185 .add(Src1);
5187 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5188 .add(Src0)
5189 .addReg(Interm);
5191 Worklist.insert(&Not);
5192 Worklist.insert(&Op);
5194 MRI.replaceRegWith(Dest.getReg(), NewDest);
5195 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5198 void SIInstrInfo::splitScalar64BitUnaryOp(
5199 SetVectorType &Worklist, MachineInstr &Inst,
5200 unsigned Opcode) const {
5201 MachineBasicBlock &MBB = *Inst.getParent();
5202 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5204 MachineOperand &Dest = Inst.getOperand(0);
5205 MachineOperand &Src0 = Inst.getOperand(1);
5206 DebugLoc DL = Inst.getDebugLoc();
5208 MachineBasicBlock::iterator MII = Inst;
5210 const MCInstrDesc &InstDesc = get(Opcode);
5211 const TargetRegisterClass *Src0RC = Src0.isReg() ?
5212 MRI.getRegClass(Src0.getReg()) :
5213 &AMDGPU::SGPR_32RegClass;
5215 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5217 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5218 AMDGPU::sub0, Src0SubRC);
5220 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5221 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5222 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5224 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5225 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5227 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5228 AMDGPU::sub1, Src0SubRC);
5230 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5231 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5233 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5234 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5235 .addReg(DestSub0)
5236 .addImm(AMDGPU::sub0)
5237 .addReg(DestSub1)
5238 .addImm(AMDGPU::sub1);
5240 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5242 Worklist.insert(&LoHalf);
5243 Worklist.insert(&HiHalf);
5245 // We don't need to legalizeOperands here because for a single operand, src0
5246 // will support any kind of input.
5248 // Move all users of this moved value.
5249 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5252 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5253 MachineInstr &Inst,
5254 MachineDominatorTree *MDT) const {
5255 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5257 MachineBasicBlock &MBB = *Inst.getParent();
5258 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5259 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5261 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5262 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5263 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5265 Register CarryReg = MRI.createVirtualRegister(CarryRC);
5266 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5268 MachineOperand &Dest = Inst.getOperand(0);
5269 MachineOperand &Src0 = Inst.getOperand(1);
5270 MachineOperand &Src1 = Inst.getOperand(2);
5271 const DebugLoc &DL = Inst.getDebugLoc();
5272 MachineBasicBlock::iterator MII = Inst;
5274 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5275 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
5276 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5277 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5279 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5280 AMDGPU::sub0, Src0SubRC);
5281 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5282 AMDGPU::sub0, Src1SubRC);
5285 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5286 AMDGPU::sub1, Src0SubRC);
5287 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5288 AMDGPU::sub1, Src1SubRC);
5290 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
5291 MachineInstr *LoHalf =
5292 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
5293 .addReg(CarryReg, RegState::Define)
5294 .add(SrcReg0Sub0)
5295 .add(SrcReg1Sub0)
5296 .addImm(0); // clamp bit
5298 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
5299 MachineInstr *HiHalf =
5300 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
5301 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
5302 .add(SrcReg0Sub1)
5303 .add(SrcReg1Sub1)
5304 .addReg(CarryReg, RegState::Kill)
5305 .addImm(0); // clamp bit
5307 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5308 .addReg(DestSub0)
5309 .addImm(AMDGPU::sub0)
5310 .addReg(DestSub1)
5311 .addImm(AMDGPU::sub1);
5313 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5315 // Try to legalize the operands in case we need to swap the order to keep it
5316 // valid.
5317 legalizeOperands(*LoHalf, MDT);
5318 legalizeOperands(*HiHalf, MDT);
5320 // Move all users of this moved vlaue.
5321 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5324 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
5325 MachineInstr &Inst, unsigned Opcode,
5326 MachineDominatorTree *MDT) const {
5327 MachineBasicBlock &MBB = *Inst.getParent();
5328 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5330 MachineOperand &Dest = Inst.getOperand(0);
5331 MachineOperand &Src0 = Inst.getOperand(1);
5332 MachineOperand &Src1 = Inst.getOperand(2);
5333 DebugLoc DL = Inst.getDebugLoc();
5335 MachineBasicBlock::iterator MII = Inst;
5337 const MCInstrDesc &InstDesc = get(Opcode);
5338 const TargetRegisterClass *Src0RC = Src0.isReg() ?
5339 MRI.getRegClass(Src0.getReg()) :
5340 &AMDGPU::SGPR_32RegClass;
5342 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5343 const TargetRegisterClass *Src1RC = Src1.isReg() ?
5344 MRI.getRegClass(Src1.getReg()) :
5345 &AMDGPU::SGPR_32RegClass;
5347 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5349 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5350 AMDGPU::sub0, Src0SubRC);
5351 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5352 AMDGPU::sub0, Src1SubRC);
5353 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5354 AMDGPU::sub1, Src0SubRC);
5355 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5356 AMDGPU::sub1, Src1SubRC);
5358 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5359 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5360 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5362 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5363 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
5364 .add(SrcReg0Sub0)
5365 .add(SrcReg1Sub0);
5367 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5368 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
5369 .add(SrcReg0Sub1)
5370 .add(SrcReg1Sub1);
5372 Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5373 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5374 .addReg(DestSub0)
5375 .addImm(AMDGPU::sub0)
5376 .addReg(DestSub1)
5377 .addImm(AMDGPU::sub1);
5379 MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5381 Worklist.insert(&LoHalf);
5382 Worklist.insert(&HiHalf);
5384 // Move all users of this moved vlaue.
5385 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5388 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist,
5389 MachineInstr &Inst,
5390 MachineDominatorTree *MDT) const {
5391 MachineBasicBlock &MBB = *Inst.getParent();
5392 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5394 MachineOperand &Dest = Inst.getOperand(0);
5395 MachineOperand &Src0 = Inst.getOperand(1);
5396 MachineOperand &Src1 = Inst.getOperand(2);
5397 const DebugLoc &DL = Inst.getDebugLoc();
5399 MachineBasicBlock::iterator MII = Inst;
5401 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5403 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5405 MachineOperand* Op0;
5406 MachineOperand* Op1;
5408 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
5409 Op0 = &Src0;
5410 Op1 = &Src1;
5411 } else {
5412 Op0 = &Src1;
5413 Op1 = &Src0;
5416 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
5417 .add(*Op0);
5419 Register NewDest = MRI.createVirtualRegister(DestRC);
5421 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
5422 .addReg(Interm)
5423 .add(*Op1);
5425 MRI.replaceRegWith(Dest.getReg(), NewDest);
5427 Worklist.insert(&Xor);
5430 void SIInstrInfo::splitScalar64BitBCNT(
5431 SetVectorType &Worklist, MachineInstr &Inst) const {
5432 MachineBasicBlock &MBB = *Inst.getParent();
5433 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5435 MachineBasicBlock::iterator MII = Inst;
5436 const DebugLoc &DL = Inst.getDebugLoc();
5438 MachineOperand &Dest = Inst.getOperand(0);
5439 MachineOperand &Src = Inst.getOperand(1);
5441 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
5442 const TargetRegisterClass *SrcRC = Src.isReg() ?
5443 MRI.getRegClass(Src.getReg()) :
5444 &AMDGPU::SGPR_32RegClass;
5446 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5447 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5449 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
5451 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5452 AMDGPU::sub0, SrcSubRC);
5453 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5454 AMDGPU::sub1, SrcSubRC);
5456 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
5458 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
5460 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5462 // We don't need to legalize operands here. src0 for etiher instruction can be
5463 // an SGPR, and the second input is unused or determined here.
5464 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5467 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
5468 MachineInstr &Inst) const {
5469 MachineBasicBlock &MBB = *Inst.getParent();
5470 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5471 MachineBasicBlock::iterator MII = Inst;
5472 const DebugLoc &DL = Inst.getDebugLoc();
5474 MachineOperand &Dest = Inst.getOperand(0);
5475 uint32_t Imm = Inst.getOperand(2).getImm();
5476 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
5477 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
5479 (void) Offset;
5481 // Only sext_inreg cases handled.
5482 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
5483 Offset == 0 && "Not implemented");
5485 if (BitWidth < 32) {
5486 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5487 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5488 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5490 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
5491 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
5492 .addImm(0)
5493 .addImm(BitWidth);
5495 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
5496 .addImm(31)
5497 .addReg(MidRegLo);
5499 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
5500 .addReg(MidRegLo)
5501 .addImm(AMDGPU::sub0)
5502 .addReg(MidRegHi)
5503 .addImm(AMDGPU::sub1);
5505 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5506 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5507 return;
5510 MachineOperand &Src = Inst.getOperand(1);
5511 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5512 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5514 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
5515 .addImm(31)
5516 .addReg(Src.getReg(), 0, AMDGPU::sub0);
5518 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
5519 .addReg(Src.getReg(), 0, AMDGPU::sub0)
5520 .addImm(AMDGPU::sub0)
5521 .addReg(TmpReg)
5522 .addImm(AMDGPU::sub1);
5524 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5525 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5528 void SIInstrInfo::addUsersToMoveToVALUWorklist(
5529 unsigned DstReg,
5530 MachineRegisterInfo &MRI,
5531 SetVectorType &Worklist) const {
5532 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
5533 E = MRI.use_end(); I != E;) {
5534 MachineInstr &UseMI = *I->getParent();
5536 unsigned OpNo = 0;
5538 switch (UseMI.getOpcode()) {
5539 case AMDGPU::COPY:
5540 case AMDGPU::WQM:
5541 case AMDGPU::SOFT_WQM:
5542 case AMDGPU::WWM:
5543 case AMDGPU::REG_SEQUENCE:
5544 case AMDGPU::PHI:
5545 case AMDGPU::INSERT_SUBREG:
5546 break;
5547 default:
5548 OpNo = I.getOperandNo();
5549 break;
5552 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) {
5553 Worklist.insert(&UseMI);
5555 do {
5556 ++I;
5557 } while (I != E && I->getParent() == &UseMI);
5558 } else {
5559 ++I;
5564 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
5565 MachineRegisterInfo &MRI,
5566 MachineInstr &Inst) const {
5567 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5568 MachineBasicBlock *MBB = Inst.getParent();
5569 MachineOperand &Src0 = Inst.getOperand(1);
5570 MachineOperand &Src1 = Inst.getOperand(2);
5571 const DebugLoc &DL = Inst.getDebugLoc();
5573 switch (Inst.getOpcode()) {
5574 case AMDGPU::S_PACK_LL_B32_B16: {
5575 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5576 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5578 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
5579 // 0.
5580 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5581 .addImm(0xffff);
5583 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
5584 .addReg(ImmReg, RegState::Kill)
5585 .add(Src0);
5587 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
5588 .add(Src1)
5589 .addImm(16)
5590 .addReg(TmpReg, RegState::Kill);
5591 break;
5593 case AMDGPU::S_PACK_LH_B32_B16: {
5594 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5595 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5596 .addImm(0xffff);
5597 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
5598 .addReg(ImmReg, RegState::Kill)
5599 .add(Src0)
5600 .add(Src1);
5601 break;
5603 case AMDGPU::S_PACK_HH_B32_B16: {
5604 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5605 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5606 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
5607 .addImm(16)
5608 .add(Src0);
5609 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
5610 .addImm(0xffff0000);
5611 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
5612 .add(Src1)
5613 .addReg(ImmReg, RegState::Kill)
5614 .addReg(TmpReg, RegState::Kill);
5615 break;
5617 default:
5618 llvm_unreachable("unhandled s_pack_* instruction");
5621 MachineOperand &Dest = Inst.getOperand(0);
5622 MRI.replaceRegWith(Dest.getReg(), ResultReg);
5623 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5626 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
5627 MachineInstr &SCCDefInst,
5628 SetVectorType &Worklist) const {
5629 // Ensure that def inst defines SCC, which is still live.
5630 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
5631 !Op.isDead() && Op.getParent() == &SCCDefInst);
5632 // This assumes that all the users of SCC are in the same block
5633 // as the SCC def.
5634 for (MachineInstr &MI : // Skip the def inst itself.
5635 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)),
5636 SCCDefInst.getParent()->end())) {
5637 // Check if SCC is used first.
5638 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1)
5639 Worklist.insert(&MI);
5640 // Exit if we find another SCC def.
5641 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
5642 return;
5646 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
5647 const MachineInstr &Inst) const {
5648 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
5650 switch (Inst.getOpcode()) {
5651 // For target instructions, getOpRegClass just returns the virtual register
5652 // class associated with the operand, so we need to find an equivalent VGPR
5653 // register class in order to move the instruction to the VALU.
5654 case AMDGPU::COPY:
5655 case AMDGPU::PHI:
5656 case AMDGPU::REG_SEQUENCE:
5657 case AMDGPU::INSERT_SUBREG:
5658 case AMDGPU::WQM:
5659 case AMDGPU::SOFT_WQM:
5660 case AMDGPU::WWM: {
5661 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1);
5662 if (RI.hasAGPRs(SrcRC)) {
5663 if (RI.hasAGPRs(NewDstRC))
5664 return nullptr;
5666 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
5667 if (!NewDstRC)
5668 return nullptr;
5669 } else {
5670 if (RI.hasVGPRs(NewDstRC))
5671 return nullptr;
5673 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
5674 if (!NewDstRC)
5675 return nullptr;
5678 return NewDstRC;
5680 default:
5681 return NewDstRC;
5685 // Find the one SGPR operand we are allowed to use.
5686 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
5687 int OpIndices[3]) const {
5688 const MCInstrDesc &Desc = MI.getDesc();
5690 // Find the one SGPR operand we are allowed to use.
5692 // First we need to consider the instruction's operand requirements before
5693 // legalizing. Some operands are required to be SGPRs, such as implicit uses
5694 // of VCC, but we are still bound by the constant bus requirement to only use
5695 // one.
5697 // If the operand's class is an SGPR, we can never move it.
5699 unsigned SGPRReg = findImplicitSGPRRead(MI);
5700 if (SGPRReg != AMDGPU::NoRegister)
5701 return SGPRReg;
5703 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
5704 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5706 for (unsigned i = 0; i < 3; ++i) {
5707 int Idx = OpIndices[i];
5708 if (Idx == -1)
5709 break;
5711 const MachineOperand &MO = MI.getOperand(Idx);
5712 if (!MO.isReg())
5713 continue;
5715 // Is this operand statically required to be an SGPR based on the operand
5716 // constraints?
5717 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
5718 bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
5719 if (IsRequiredSGPR)
5720 return MO.getReg();
5722 // If this could be a VGPR or an SGPR, Check the dynamic register class.
5723 Register Reg = MO.getReg();
5724 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
5725 if (RI.isSGPRClass(RegRC))
5726 UsedSGPRs[i] = Reg;
5729 // We don't have a required SGPR operand, so we have a bit more freedom in
5730 // selecting operands to move.
5732 // Try to select the most used SGPR. If an SGPR is equal to one of the
5733 // others, we choose that.
5735 // e.g.
5736 // V_FMA_F32 v0, s0, s0, s0 -> No moves
5737 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
5739 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
5740 // prefer those.
5742 if (UsedSGPRs[0] != AMDGPU::NoRegister) {
5743 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
5744 SGPRReg = UsedSGPRs[0];
5747 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
5748 if (UsedSGPRs[1] == UsedSGPRs[2])
5749 SGPRReg = UsedSGPRs[1];
5752 return SGPRReg;
5755 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
5756 unsigned OperandName) const {
5757 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
5758 if (Idx == -1)
5759 return nullptr;
5761 return &MI.getOperand(Idx);
5764 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
5765 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5766 return (22ULL << 44) | // IMG_FORMAT_32_FLOAT
5767 (1ULL << 56) | // RESOURCE_LEVEL = 1
5768 (3ULL << 60); // OOB_SELECT = 3
5771 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
5772 if (ST.isAmdHsaOS()) {
5773 // Set ATC = 1. GFX9 doesn't have this bit.
5774 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5775 RsrcDataFormat |= (1ULL << 56);
5777 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
5778 // BTW, it disables TC L2 and therefore decreases performance.
5779 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
5780 RsrcDataFormat |= (2ULL << 59);
5783 return RsrcDataFormat;
5786 uint64_t SIInstrInfo::getScratchRsrcWords23() const {
5787 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
5788 AMDGPU::RSRC_TID_ENABLE |
5789 0xffffffff; // Size;
5791 // GFX9 doesn't have ELEMENT_SIZE.
5792 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5793 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
5794 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
5797 // IndexStride = 64 / 32.
5798 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2;
5799 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
5801 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
5802 // Clear them unless we want a huge stride.
5803 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
5804 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
5805 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
5807 return Rsrc23;
5810 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
5811 unsigned Opc = MI.getOpcode();
5813 return isSMRD(Opc);
5816 bool SIInstrInfo::isHighLatencyInstruction(const MachineInstr &MI) const {
5817 unsigned Opc = MI.getOpcode();
5819 return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
5822 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
5823 int &FrameIndex) const {
5824 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
5825 if (!Addr || !Addr->isFI())
5826 return AMDGPU::NoRegister;
5828 assert(!MI.memoperands_empty() &&
5829 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
5831 FrameIndex = Addr->getIndex();
5832 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
5835 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
5836 int &FrameIndex) const {
5837 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
5838 assert(Addr && Addr->isFI());
5839 FrameIndex = Addr->getIndex();
5840 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
5843 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
5844 int &FrameIndex) const {
5845 if (!MI.mayLoad())
5846 return AMDGPU::NoRegister;
5848 if (isMUBUF(MI) || isVGPRSpill(MI))
5849 return isStackAccess(MI, FrameIndex);
5851 if (isSGPRSpill(MI))
5852 return isSGPRStackAccess(MI, FrameIndex);
5854 return AMDGPU::NoRegister;
5857 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
5858 int &FrameIndex) const {
5859 if (!MI.mayStore())
5860 return AMDGPU::NoRegister;
5862 if (isMUBUF(MI) || isVGPRSpill(MI))
5863 return isStackAccess(MI, FrameIndex);
5865 if (isSGPRSpill(MI))
5866 return isSGPRStackAccess(MI, FrameIndex);
5868 return AMDGPU::NoRegister;
5871 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
5872 unsigned Size = 0;
5873 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
5874 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
5875 while (++I != E && I->isInsideBundle()) {
5876 assert(!I->isBundle() && "No nested bundle!");
5877 Size += getInstSizeInBytes(*I);
5880 return Size;
5883 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
5884 unsigned Opc = MI.getOpcode();
5885 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
5886 unsigned DescSize = Desc.getSize();
5888 // If we have a definitive size, we can use it. Otherwise we need to inspect
5889 // the operands to know the size.
5890 if (isFixedSize(MI))
5891 return DescSize;
5893 // 4-byte instructions may have a 32-bit literal encoded after them. Check
5894 // operands that coud ever be literals.
5895 if (isVALU(MI) || isSALU(MI)) {
5896 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
5897 if (Src0Idx == -1)
5898 return DescSize; // No operands.
5900 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
5901 return isVOP3(MI) ? 12 : (DescSize + 4);
5903 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5904 if (Src1Idx == -1)
5905 return DescSize;
5907 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
5908 return isVOP3(MI) ? 12 : (DescSize + 4);
5910 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
5911 if (Src2Idx == -1)
5912 return DescSize;
5914 if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx]))
5915 return isVOP3(MI) ? 12 : (DescSize + 4);
5917 return DescSize;
5920 // Check whether we have extra NSA words.
5921 if (isMIMG(MI)) {
5922 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
5923 if (VAddr0Idx < 0)
5924 return 8;
5926 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
5927 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
5930 switch (Opc) {
5931 case TargetOpcode::IMPLICIT_DEF:
5932 case TargetOpcode::KILL:
5933 case TargetOpcode::DBG_VALUE:
5934 case TargetOpcode::EH_LABEL:
5935 return 0;
5936 case TargetOpcode::BUNDLE:
5937 return getInstBundleSize(MI);
5938 case TargetOpcode::INLINEASM:
5939 case TargetOpcode::INLINEASM_BR: {
5940 const MachineFunction *MF = MI.getParent()->getParent();
5941 const char *AsmStr = MI.getOperand(0).getSymbolName();
5942 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(),
5943 &MF->getSubtarget());
5945 default:
5946 return DescSize;
5950 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
5951 if (!isFLAT(MI))
5952 return false;
5954 if (MI.memoperands_empty())
5955 return true;
5957 for (const MachineMemOperand *MMO : MI.memoperands()) {
5958 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
5959 return true;
5961 return false;
5964 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
5965 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
5968 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
5969 MachineBasicBlock *IfEnd) const {
5970 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
5971 assert(TI != IfEntry->end());
5973 MachineInstr *Branch = &(*TI);
5974 MachineFunction *MF = IfEntry->getParent();
5975 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
5977 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
5978 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
5979 MachineInstr *SIIF =
5980 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
5981 .add(Branch->getOperand(0))
5982 .add(Branch->getOperand(1));
5983 MachineInstr *SIEND =
5984 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
5985 .addReg(DstReg);
5987 IfEntry->erase(TI);
5988 IfEntry->insert(IfEntry->end(), SIIF);
5989 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
5993 void SIInstrInfo::convertNonUniformLoopRegion(
5994 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
5995 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
5996 // We expect 2 terminators, one conditional and one unconditional.
5997 assert(TI != LoopEnd->end());
5999 MachineInstr *Branch = &(*TI);
6000 MachineFunction *MF = LoopEnd->getParent();
6001 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
6003 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
6005 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
6006 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC());
6007 MachineInstrBuilder HeaderPHIBuilder =
6008 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
6009 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(),
6010 E = LoopEntry->pred_end();
6011 PI != E; ++PI) {
6012 if (*PI == LoopEnd) {
6013 HeaderPHIBuilder.addReg(BackEdgeReg);
6014 } else {
6015 MachineBasicBlock *PMBB = *PI;
6016 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC());
6017 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(),
6018 ZeroReg, 0);
6019 HeaderPHIBuilder.addReg(ZeroReg);
6021 HeaderPHIBuilder.addMBB(*PI);
6023 MachineInstr *HeaderPhi = HeaderPHIBuilder;
6024 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
6025 get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
6026 .addReg(DstReg)
6027 .add(Branch->getOperand(0));
6028 MachineInstr *SILOOP =
6029 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
6030 .addReg(BackEdgeReg)
6031 .addMBB(LoopEntry);
6033 LoopEntry->insert(LoopEntry->begin(), HeaderPhi);
6034 LoopEnd->erase(TI);
6035 LoopEnd->insert(LoopEnd->end(), SIIFBREAK);
6036 LoopEnd->insert(LoopEnd->end(), SILOOP);
6040 ArrayRef<std::pair<int, const char *>>
6041 SIInstrInfo::getSerializableTargetIndices() const {
6042 static const std::pair<int, const char *> TargetIndices[] = {
6043 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
6044 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
6045 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
6046 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
6047 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
6048 return makeArrayRef(TargetIndices);
6051 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
6052 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
6053 ScheduleHazardRecognizer *
6054 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
6055 const ScheduleDAG *DAG) const {
6056 return new GCNHazardRecognizer(DAG->MF);
6059 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
6060 /// pass.
6061 ScheduleHazardRecognizer *
6062 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
6063 return new GCNHazardRecognizer(MF);
6066 std::pair<unsigned, unsigned>
6067 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
6068 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK);
6071 ArrayRef<std::pair<unsigned, const char *>>
6072 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
6073 static const std::pair<unsigned, const char *> TargetFlags[] = {
6074 { MO_GOTPCREL, "amdgpu-gotprel" },
6075 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
6076 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
6077 { MO_REL32_LO, "amdgpu-rel32-lo" },
6078 { MO_REL32_HI, "amdgpu-rel32-hi" },
6079 { MO_ABS32_LO, "amdgpu-abs32-lo" },
6080 { MO_ABS32_HI, "amdgpu-abs32-hi" },
6083 return makeArrayRef(TargetFlags);
6086 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const {
6087 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
6088 MI.modifiesRegister(AMDGPU::EXEC, &RI);
6091 MachineInstrBuilder
6092 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
6093 MachineBasicBlock::iterator I,
6094 const DebugLoc &DL,
6095 unsigned DestReg) const {
6096 if (ST.hasAddNoCarry())
6097 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
6099 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
6100 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC());
6101 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC());
6103 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
6104 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
6107 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
6108 MachineBasicBlock::iterator I,
6109 const DebugLoc &DL,
6110 Register DestReg,
6111 RegScavenger &RS) const {
6112 if (ST.hasAddNoCarry())
6113 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);
6115 Register UnusedCarry = RS.scavengeRegister(RI.getBoolRC(), I, 0, false);
6116 // TODO: Users need to deal with this.
6117 if (!UnusedCarry.isValid())
6118 return MachineInstrBuilder();
6120 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
6121 .addReg(UnusedCarry, RegState::Define | RegState::Dead);
6124 bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
6125 switch (Opcode) {
6126 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
6127 case AMDGPU::SI_KILL_I1_TERMINATOR:
6128 return true;
6129 default:
6130 return false;
6134 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
6135 switch (Opcode) {
6136 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
6137 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
6138 case AMDGPU::SI_KILL_I1_PSEUDO:
6139 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
6140 default:
6141 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
6145 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
6146 MachineBasicBlock *MBB = MI.getParent();
6147 MachineFunction *MF = MBB->getParent();
6148 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6150 if (!ST.isWave32())
6151 return;
6153 for (auto &Op : MI.implicit_operands()) {
6154 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
6155 Op.setReg(AMDGPU::VCC_LO);
6159 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
6160 if (!isSMRD(MI))
6161 return false;
6163 // Check that it is using a buffer resource.
6164 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
6165 if (Idx == -1) // e.g. s_memtime
6166 return false;
6168 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass;
6169 return RCID == AMDGPU::SReg_128RegClassID;
6172 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
6173 bool Signed) const {
6174 // TODO: Should 0 be special cased?
6175 if (!ST.hasFlatInstOffsets())
6176 return false;
6178 if (ST.hasFlatSegmentOffsetBug() && AddrSpace == AMDGPUAS::FLAT_ADDRESS)
6179 return false;
6181 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
6182 return (Signed && isInt<12>(Offset)) ||
6183 (!Signed && isUInt<11>(Offset));
6186 return (Signed && isInt<13>(Offset)) ||
6187 (!Signed && isUInt<12>(Offset));
6191 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
6192 enum SIEncodingFamily {
6193 SI = 0,
6194 VI = 1,
6195 SDWA = 2,
6196 SDWA9 = 3,
6197 GFX80 = 4,
6198 GFX9 = 5,
6199 GFX10 = 6,
6200 SDWA10 = 7
6203 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) {
6204 switch (ST.getGeneration()) {
6205 default:
6206 break;
6207 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
6208 case AMDGPUSubtarget::SEA_ISLANDS:
6209 return SIEncodingFamily::SI;
6210 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
6211 case AMDGPUSubtarget::GFX9:
6212 return SIEncodingFamily::VI;
6213 case AMDGPUSubtarget::GFX10:
6214 return SIEncodingFamily::GFX10;
6216 llvm_unreachable("Unknown subtarget generation!");
6219 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
6220 SIEncodingFamily Gen = subtargetEncodingFamily(ST);
6222 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
6223 ST.getGeneration() == AMDGPUSubtarget::GFX9)
6224 Gen = SIEncodingFamily::GFX9;
6226 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
6227 // subtarget has UnpackedD16VMem feature.
6228 // TODO: remove this when we discard GFX80 encoding.
6229 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
6230 Gen = SIEncodingFamily::GFX80;
6232 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) {
6233 switch (ST.getGeneration()) {
6234 default:
6235 Gen = SIEncodingFamily::SDWA;
6236 break;
6237 case AMDGPUSubtarget::GFX9:
6238 Gen = SIEncodingFamily::SDWA9;
6239 break;
6240 case AMDGPUSubtarget::GFX10:
6241 Gen = SIEncodingFamily::SDWA10;
6242 break;
6246 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
6248 // -1 means that Opcode is already a native instruction.
6249 if (MCOp == -1)
6250 return Opcode;
6252 // (uint16_t)-1 means that Opcode is a pseudo instruction that has
6253 // no encoding in the given subtarget generation.
6254 if (MCOp == (uint16_t)-1)
6255 return -1;
6257 return MCOp;
6260 static
6261 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
6262 assert(RegOpnd.isReg());
6263 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
6264 getRegSubRegPair(RegOpnd);
6267 TargetInstrInfo::RegSubRegPair
6268 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
6269 assert(MI.isRegSequence());
6270 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
6271 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) {
6272 auto &RegOp = MI.getOperand(1 + 2 * I);
6273 return getRegOrUndef(RegOp);
6275 return TargetInstrInfo::RegSubRegPair();
6278 // Try to find the definition of reg:subreg in subreg-manipulation pseudos
6279 // Following a subreg of reg:subreg isn't supported
6280 static bool followSubRegDef(MachineInstr &MI,
6281 TargetInstrInfo::RegSubRegPair &RSR) {
6282 if (!RSR.SubReg)
6283 return false;
6284 switch (MI.getOpcode()) {
6285 default: break;
6286 case AMDGPU::REG_SEQUENCE:
6287 RSR = getRegSequenceSubReg(MI, RSR.SubReg);
6288 return true;
6289 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
6290 case AMDGPU::INSERT_SUBREG:
6291 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm())
6292 // inserted the subreg we're looking for
6293 RSR = getRegOrUndef(MI.getOperand(2));
6294 else { // the subreg in the rest of the reg
6295 auto R1 = getRegOrUndef(MI.getOperand(1));
6296 if (R1.SubReg) // subreg of subreg isn't supported
6297 return false;
6298 RSR.Reg = R1.Reg;
6300 return true;
6302 return false;
6305 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
6306 MachineRegisterInfo &MRI) {
6307 assert(MRI.isSSA());
6308 if (!Register::isVirtualRegister(P.Reg))
6309 return nullptr;
6311 auto RSR = P;
6312 auto *DefInst = MRI.getVRegDef(RSR.Reg);
6313 while (auto *MI = DefInst) {
6314 DefInst = nullptr;
6315 switch (MI->getOpcode()) {
6316 case AMDGPU::COPY:
6317 case AMDGPU::V_MOV_B32_e32: {
6318 auto &Op1 = MI->getOperand(1);
6319 if (Op1.isReg() && Register::isVirtualRegister(Op1.getReg())) {
6320 if (Op1.isUndef())
6321 return nullptr;
6322 RSR = getRegSubRegPair(Op1);
6323 DefInst = MRI.getVRegDef(RSR.Reg);
6325 break;
6327 default:
6328 if (followSubRegDef(*MI, RSR)) {
6329 if (!RSR.Reg)
6330 return nullptr;
6331 DefInst = MRI.getVRegDef(RSR.Reg);
6334 if (!DefInst)
6335 return MI;
6337 return nullptr;
6340 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
6341 Register VReg,
6342 const MachineInstr &DefMI,
6343 const MachineInstr &UseMI) {
6344 assert(MRI.isSSA() && "Must be run on SSA");
6346 auto *TRI = MRI.getTargetRegisterInfo();
6347 auto *DefBB = DefMI.getParent();
6349 // Don't bother searching between blocks, although it is possible this block
6350 // doesn't modify exec.
6351 if (UseMI.getParent() != DefBB)
6352 return true;
6354 const int MaxInstScan = 20;
6355 int NumInst = 0;
6357 // Stop scan at the use.
6358 auto E = UseMI.getIterator();
6359 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) {
6360 if (I->isDebugInstr())
6361 continue;
6363 if (++NumInst > MaxInstScan)
6364 return true;
6366 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
6367 return true;
6370 return false;
6373 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
6374 Register VReg,
6375 const MachineInstr &DefMI) {
6376 assert(MRI.isSSA() && "Must be run on SSA");
6378 auto *TRI = MRI.getTargetRegisterInfo();
6379 auto *DefBB = DefMI.getParent();
6381 const int MaxUseInstScan = 10;
6382 int NumUseInst = 0;
6384 for (auto &UseInst : MRI.use_nodbg_instructions(VReg)) {
6385 // Don't bother searching between blocks, although it is possible this block
6386 // doesn't modify exec.
6387 if (UseInst.getParent() != DefBB)
6388 return true;
6390 if (++NumUseInst > MaxUseInstScan)
6391 return true;
6394 const int MaxInstScan = 20;
6395 int NumInst = 0;
6397 // Stop scan when we have seen all the uses.
6398 for (auto I = std::next(DefMI.getIterator()); ; ++I) {
6399 if (I->isDebugInstr())
6400 continue;
6402 if (++NumInst > MaxInstScan)
6403 return true;
6405 if (I->readsRegister(VReg))
6406 if (--NumUseInst == 0)
6407 return false;
6409 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
6410 return true;
6414 MachineInstr *SIInstrInfo::createPHIDestinationCopy(
6415 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
6416 const DebugLoc &DL, Register Src, Register Dst) const {
6417 auto Cur = MBB.begin();
6418 if (Cur != MBB.end())
6419 do {
6420 if (!Cur->isPHI() && Cur->readsRegister(Dst))
6421 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);
6422 ++Cur;
6423 } while (Cur != MBB.end() && Cur != LastPHIIt);
6425 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,
6426 Dst);
6429 MachineInstr *SIInstrInfo::createPHISourceCopy(
6430 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
6431 const DebugLoc &DL, Register Src, Register SrcSubReg, Register Dst) const {
6432 if (InsPt != MBB.end() &&
6433 (InsPt->getOpcode() == AMDGPU::SI_IF ||
6434 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
6435 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
6436 InsPt->definesRegister(Src)) {
6437 InsPt++;
6438 return BuildMI(MBB, InsPt, InsPt->getDebugLoc(),
6439 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term
6440 : AMDGPU::S_MOV_B64_term),
6441 Dst)
6442 .addReg(Src, 0, SrcSubReg)
6443 .addReg(AMDGPU::EXEC, RegState::Implicit);
6445 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
6446 Dst);
6449 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }