?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[Clang/AMDGPU] Zero sized arrays not allowed in HIP device code. (#113470)
[llvm-project.git] / llvm / lib / Target / AMDGPU / SIFoldOperands.cpp
blob4fb5cb066be7c30d5526e15c6b9affe75bdc8628
1 //===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 /// \file
8 //===----------------------------------------------------------------------===//
9 //
10
11 #include "SIFoldOperands.h"
12 #include "AMDGPU.h"
13 #include "GCNSubtarget.h"
14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15 #include "SIMachineFunctionInfo.h"
16 #include "llvm/ADT/DepthFirstIterator.h"
17 #include "llvm/CodeGen/MachineFunctionPass.h"
18 #include "llvm/CodeGen/MachineOperand.h"
19
20 #define DEBUG_TYPE "si-fold-operands"
21 using namespace llvm;
22
23 namespace {
24
25 struct FoldCandidate {
26 MachineInstr *UseMI;
27 union {
28 MachineOperand *OpToFold;
29 uint64_t ImmToFold;
30 int FrameIndexToFold;
31 };
32 int ShrinkOpcode;
33 unsigned UseOpNo;
34 MachineOperand::MachineOperandType Kind;
35 bool Commuted;
36
37 FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp,
38 bool Commuted_ = false,
39 int ShrinkOp = -1) :
40 UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
41 Kind(FoldOp->getType()),
42 Commuted(Commuted_) {
43 if (FoldOp->isImm()) {
44 ImmToFold = FoldOp->getImm();
45 } else if (FoldOp->isFI()) {
46 FrameIndexToFold = FoldOp->getIndex();
47 } else {
48 assert(FoldOp->isReg() || FoldOp->isGlobal());
49 OpToFold = FoldOp;
50 }
51 }
52
53 bool isFI() const {
54 return Kind == MachineOperand::MO_FrameIndex;
55 }
56
57 bool isImm() const {
58 return Kind == MachineOperand::MO_Immediate;
59 }
60
61 bool isReg() const {
62 return Kind == MachineOperand::MO_Register;
63 }
64
65 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
66
67 bool needsShrink() const { return ShrinkOpcode != -1; }
68 };
69
70 class SIFoldOperandsImpl {
71 public:
72 MachineRegisterInfo *MRI;
73 const SIInstrInfo *TII;
74 const SIRegisterInfo *TRI;
75 const GCNSubtarget *ST;
76 const SIMachineFunctionInfo *MFI;
77
78 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
79 const MachineOperand &OpToFold) const;
80
81 // TODO: Just use TII::getVALUOp
82 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
83 switch (Opc) {
84 case AMDGPU::S_ADD_I32: {
85 if (ST->hasAddNoCarry())
86 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
87 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
88 }
89 case AMDGPU::S_OR_B32:
90 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
91 case AMDGPU::S_AND_B32:
92 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
93 case AMDGPU::S_MUL_I32:
94 return AMDGPU::V_MUL_LO_U32_e64;
95 default:
96 return AMDGPU::INSTRUCTION_LIST_END;
97 }
98 }
99
100 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
101 MachineInstr &MI) const;
102
103 bool updateOperand(FoldCandidate &Fold) const;
104
105 bool canUseImmWithOpSel(FoldCandidate &Fold) const;
106
107 bool tryFoldImmWithOpSel(FoldCandidate &Fold) const;
108
109 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
110 MachineInstr *MI, unsigned OpNo,
111 MachineOperand *OpToFold) const;
112 bool isUseSafeToFold(const MachineInstr &MI,
113 const MachineOperand &UseMO) const;
114 bool
115 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
116 Register UseReg, uint8_t OpTy) const;
117 bool tryToFoldACImm(const MachineOperand &OpToFold, MachineInstr *UseMI,
118 unsigned UseOpIdx,
119 SmallVectorImpl<FoldCandidate> &FoldList) const;
120 void foldOperand(MachineOperand &OpToFold,
121 MachineInstr *UseMI,
122 int UseOpIdx,
123 SmallVectorImpl<FoldCandidate> &FoldList,
124 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
125
126 MachineOperand *getImmOrMaterializedImm(MachineOperand &Op) const;
127 bool tryConstantFoldOp(MachineInstr *MI) const;
128 bool tryFoldCndMask(MachineInstr &MI) const;
129 bool tryFoldZeroHighBits(MachineInstr &MI) const;
130 bool foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
131 bool tryFoldFoldableCopy(MachineInstr &MI,
132 MachineOperand *&CurrentKnownM0Val) const;
133
134 const MachineOperand *isClamp(const MachineInstr &MI) const;
135 bool tryFoldClamp(MachineInstr &MI);
136
137 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
138 bool tryFoldOMod(MachineInstr &MI);
139 bool tryFoldRegSequence(MachineInstr &MI);
140 bool tryFoldPhiAGPR(MachineInstr &MI);
141 bool tryFoldLoad(MachineInstr &MI);
142
143 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
144
145 public:
146 SIFoldOperandsImpl() = default;
147
148 bool run(MachineFunction &MF);
149 };
150
151 class SIFoldOperandsLegacy : public MachineFunctionPass {
152 public:
153 static char ID;
154
155 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
156
157 bool runOnMachineFunction(MachineFunction &MF) override {
158 if (skipFunction(MF.getFunction()))
159 return false;
160 return SIFoldOperandsImpl().run(MF);
161 }
162
163 StringRef getPassName() const override { return "SI Fold Operands"; }
164
165 void getAnalysisUsage(AnalysisUsage &AU) const override {
166 AU.setPreservesCFG();
167 MachineFunctionPass::getAnalysisUsage(AU);
168 }
169 };
170
171 } // End anonymous namespace.
172
173 INITIALIZE_PASS(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
174 false)
175
176 char SIFoldOperandsLegacy::ID = 0;
177
178 char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
179
180 static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
181 const TargetRegisterInfo &TRI,
182 const MachineOperand &MO) {
183 const TargetRegisterClass *RC = MRI.getRegClass(MO.getReg());
184 if (const TargetRegisterClass *SubRC =
185 TRI.getSubRegisterClass(RC, MO.getSubReg()))
186 RC = SubRC;
187 return RC;
188 }
189
190 // Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
191 static unsigned macToMad(unsigned Opc) {
192 switch (Opc) {
193 case AMDGPU::V_MAC_F32_e64:
194 return AMDGPU::V_MAD_F32_e64;
195 case AMDGPU::V_MAC_F16_e64:
196 return AMDGPU::V_MAD_F16_e64;
197 case AMDGPU::V_FMAC_F32_e64:
198 return AMDGPU::V_FMA_F32_e64;
199 case AMDGPU::V_FMAC_F16_e64:
200 return AMDGPU::V_FMA_F16_gfx9_e64;
201 case AMDGPU::V_FMAC_F16_fake16_e64:
202 return AMDGPU::V_FMA_F16_gfx9_e64;
203 case AMDGPU::V_FMAC_LEGACY_F32_e64:
204 return AMDGPU::V_FMA_LEGACY_F32_e64;
205 case AMDGPU::V_FMAC_F64_e64:
206 return AMDGPU::V_FMA_F64_e64;
207 }
208 return AMDGPU::INSTRUCTION_LIST_END;
209 }
210
211 // TODO: Add heuristic that the frame index might not fit in the addressing mode
212 // immediate offset to avoid materializing in loops.
213 bool SIFoldOperandsImpl::frameIndexMayFold(
214 const MachineInstr &UseMI, int OpNo, const MachineOperand &OpToFold) const {
215 if (!OpToFold.isFI())
216 return false;
217
218 const unsigned Opc = UseMI.getOpcode();
219 switch (Opc) {
220 case AMDGPU::S_ADD_I32:
221 case AMDGPU::S_OR_B32:
222 case AMDGPU::S_AND_B32:
223 case AMDGPU::V_ADD_U32_e32:
224 case AMDGPU::V_ADD_CO_U32_e32:
225 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
226 // to insert the wave size shift at every point we use the index.
227 // TODO: Fix depending on visit order to fold immediates into the operand
228 return UseMI.getOperand(OpNo == 1 ? 2 : 1).isImm() &&
229 MRI->hasOneNonDBGUse(UseMI.getOperand(OpNo).getReg());
230 case AMDGPU::V_ADD_U32_e64:
231 case AMDGPU::V_ADD_CO_U32_e64:
232 return UseMI.getOperand(OpNo == 2 ? 3 : 2).isImm() &&
233 MRI->hasOneNonDBGUse(UseMI.getOperand(OpNo).getReg());
234 default:
235 break;
236 }
237
238 if (TII->isMUBUF(UseMI))
239 return OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
240 if (!TII->isFLATScratch(UseMI))
241 return false;
242
243 int SIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
244 if (OpNo == SIdx)
245 return true;
246
247 int VIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
248 return OpNo == VIdx && SIdx == -1;
249 }
250
251 /// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
252 ///
253 /// => %vgpr = V_ADD_U32 x, frameindex
254 bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
255 Register DstReg, Register SrcReg, MachineInstr &MI) const {
256 if (TRI->isVGPR(*MRI, DstReg) && TRI->isSGPRReg(*MRI, SrcReg) &&
257 MRI->hasOneNonDBGUse(SrcReg)) {
258 MachineInstr *Def = MRI->getVRegDef(SrcReg);
259 if (!Def || Def->getNumOperands() != 4)
260 return false;
261
262 MachineOperand *Src0 = &Def->getOperand(1);
263 MachineOperand *Src1 = &Def->getOperand(2);
264
265 // TODO: This is profitable with more operand types, and for more
266 // opcodes. But ultimately this is working around poor / nonexistent
267 // regbankselect.
268 if (!Src0->isFI() && !Src1->isFI())
269 return false;
270
271 if (Src0->isFI())
272 std::swap(Src0, Src1);
273
274 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(*Src0);
275 unsigned NewOp = convertToVALUOp(Def->getOpcode(), UseVOP3);
276 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
277 !Def->getOperand(3).isDead()) // Check if scc is dead
278 return false;
279
280 MachineBasicBlock *MBB = Def->getParent();
281 const DebugLoc &DL = Def->getDebugLoc();
282 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
283 MachineInstrBuilder Add =
284 BuildMI(*MBB, *Def, DL, TII->get(NewOp), DstReg);
285
286 if (Add->getDesc().getNumDefs() == 2) {
287 Register CarryOutReg = MRI->createVirtualRegister(TRI->getBoolRC());
288 Add.addDef(CarryOutReg, RegState::Dead);
289 MRI->setRegAllocationHint(CarryOutReg, 0, TRI->getVCC());
290 }
291
292 Add.add(*Src0).add(*Src1).setMIFlags(Def->getFlags());
293 if (AMDGPU::hasNamedOperand(NewOp, AMDGPU::OpName::clamp))
294 Add.addImm(0);
295
296 Def->eraseFromParent();
297 MI.eraseFromParent();
298 return true;
299 }
300
301 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
302
303 MachineBasicBlock::LivenessQueryResult Liveness =
304 MBB->computeRegisterLiveness(TRI, AMDGPU::VCC, *Def, 16);
305 if (Liveness == MachineBasicBlock::LQR_Dead) {
306 // TODO: If src1 satisfies operand constraints, use vop3 version.
307 BuildMI(*MBB, *Def, DL, TII->get(NewOp), DstReg)
308 .add(*Src0)
309 .add(*Src1)
310 .setOperandDead(3) // implicit-def $vcc
311 .setMIFlags(Def->getFlags());
312 Def->eraseFromParent();
313 MI.eraseFromParent();
314 return true;
315 }
316 }
317
318 return false;
319 }
320
321 FunctionPass *llvm::createSIFoldOperandsLegacyPass() {
322 return new SIFoldOperandsLegacy();
323 }
324
325 bool SIFoldOperandsImpl::canUseImmWithOpSel(FoldCandidate &Fold) const {
326 MachineInstr *MI = Fold.UseMI;
327 MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
328 const uint64_t TSFlags = MI->getDesc().TSFlags;
329
330 assert(Old.isReg() && Fold.isImm());
331
332 if (!(TSFlags & SIInstrFlags::IsPacked) || (TSFlags & SIInstrFlags::IsMAI) ||
333 (TSFlags & SIInstrFlags::IsWMMA) || (TSFlags & SIInstrFlags::IsSWMMAC) ||
334 (ST->hasDOTOpSelHazard() && (TSFlags & SIInstrFlags::IsDOT)))
335 return false;
336
337 unsigned Opcode = MI->getOpcode();
338 int OpNo = MI->getOperandNo(&Old);
339 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
340 switch (OpType) {
341 default:
342 return false;
343 case AMDGPU::OPERAND_REG_IMM_V2FP16:
344 case AMDGPU::OPERAND_REG_IMM_V2BF16:
345 case AMDGPU::OPERAND_REG_IMM_V2INT16:
346 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
347 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
348 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
349 break;
350 }
351
352 return true;
353 }
354
355 bool SIFoldOperandsImpl::tryFoldImmWithOpSel(FoldCandidate &Fold) const {
356 MachineInstr *MI = Fold.UseMI;
357 MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
358 unsigned Opcode = MI->getOpcode();
359 int OpNo = MI->getOperandNo(&Old);
360 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
361
362 // If the literal can be inlined as-is, apply it and short-circuit the
363 // tests below. The main motivation for this is to avoid unintuitive
364 // uses of opsel.
365 if (AMDGPU::isInlinableLiteralV216(Fold.ImmToFold, OpType)) {
366 Old.ChangeToImmediate(Fold.ImmToFold);
367 return true;
368 }
369
370 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
371 // op_sel in a way that allows an inline constant.
372 int ModIdx = -1;
373 unsigned SrcIdx = ~0;
374 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0)) {
375 ModIdx = AMDGPU::OpName::src0_modifiers;
376 SrcIdx = 0;
377 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1)) {
378 ModIdx = AMDGPU::OpName::src1_modifiers;
379 SrcIdx = 1;
380 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2)) {
381 ModIdx = AMDGPU::OpName::src2_modifiers;
382 SrcIdx = 2;
383 }
384 assert(ModIdx != -1);
385 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx);
386 MachineOperand &Mod = MI->getOperand(ModIdx);
387 unsigned ModVal = Mod.getImm();
388
389 uint16_t ImmLo = static_cast<uint16_t>(
390 Fold.ImmToFold >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
391 uint16_t ImmHi = static_cast<uint16_t>(
392 Fold.ImmToFold >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
393 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
394 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
395
396 // Helper function that attempts to inline the given value with a newly
397 // chosen opsel pattern.
398 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
399 if (AMDGPU::isInlinableLiteralV216(Imm, OpType)) {
400 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
401 Old.ChangeToImmediate(Imm);
402 return true;
403 }
404
405 // Try to shuffle the halves around and leverage opsel to get an inline
406 // constant.
407 uint16_t Lo = static_cast<uint16_t>(Imm);
408 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
409 if (Lo == Hi) {
410 if (AMDGPU::isInlinableLiteralV216(Lo, OpType)) {
411 Mod.setImm(NewModVal);
412 Old.ChangeToImmediate(Lo);
413 return true;
414 }
415
416 if (static_cast<int16_t>(Lo) < 0) {
417 int32_t SExt = static_cast<int16_t>(Lo);
418 if (AMDGPU::isInlinableLiteralV216(SExt, OpType)) {
419 Mod.setImm(NewModVal);
420 Old.ChangeToImmediate(SExt);
421 return true;
422 }
423 }
424
425 // This check is only useful for integer instructions
426 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16 ||
427 OpType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16) {
428 if (AMDGPU::isInlinableLiteralV216(Lo << 16, OpType)) {
429 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
430 Old.ChangeToImmediate(static_cast<uint32_t>(Lo) << 16);
431 return true;
432 }
433 }
434 } else {
435 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
436 if (AMDGPU::isInlinableLiteralV216(Swapped, OpType)) {
437 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
438 Old.ChangeToImmediate(Swapped);
439 return true;
440 }
441 }
442
443 return false;
444 };
445
446 if (tryFoldToInline(Imm))
447 return true;
448
449 // Replace integer addition by subtraction and vice versa if it allows
450 // folding the immediate to an inline constant.
451 //
452 // We should only ever get here for SrcIdx == 1 due to canonicalization
453 // earlier in the pipeline, but we double-check here to be safe / fully
454 // general.
455 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
456 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
457 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
458 unsigned ClampIdx =
459 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::clamp);
460 bool Clamp = MI->getOperand(ClampIdx).getImm() != 0;
461
462 if (!Clamp) {
463 uint16_t NegLo = -static_cast<uint16_t>(Imm);
464 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
465 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
466
467 if (tryFoldToInline(NegImm)) {
468 unsigned NegOpcode =
469 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
470 MI->setDesc(TII->get(NegOpcode));
471 return true;
472 }
473 }
474 }
475
476 return false;
477 }
478
479 bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
480 MachineInstr *MI = Fold.UseMI;
481 MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
482 assert(Old.isReg());
483
484 if (Fold.isImm() && canUseImmWithOpSel(Fold)) {
485 if (tryFoldImmWithOpSel(Fold))
486 return true;
487
488 // We can't represent the candidate as an inline constant. Try as a literal
489 // with the original opsel, checking constant bus limitations.
490 MachineOperand New = MachineOperand::CreateImm(Fold.ImmToFold);
491 int OpNo = MI->getOperandNo(&Old);
492 if (!TII->isOperandLegal(*MI, OpNo, &New))
493 return false;
494 Old.ChangeToImmediate(Fold.ImmToFold);
495 return true;
496 }
497
498 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
499 MachineBasicBlock *MBB = MI->getParent();
500 auto Liveness = MBB->computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 16);
501 if (Liveness != MachineBasicBlock::LQR_Dead) {
502 LLVM_DEBUG(dbgs() << "Not shrinking " << MI << " due to vcc liveness\n");
503 return false;
504 }
505
506 int Op32 = Fold.ShrinkOpcode;
507 MachineOperand &Dst0 = MI->getOperand(0);
508 MachineOperand &Dst1 = MI->getOperand(1);
509 assert(Dst0.isDef() && Dst1.isDef());
510
511 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(Dst1.getReg());
512
513 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Dst0.getReg());
514 Register NewReg0 = MRI->createVirtualRegister(Dst0RC);
515
516 MachineInstr *Inst32 = TII->buildShrunkInst(*MI, Op32);
517
518 if (HaveNonDbgCarryUse) {
519 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::COPY),
520 Dst1.getReg())
521 .addReg(AMDGPU::VCC, RegState::Kill);
522 }
523
524 // Keep the old instruction around to avoid breaking iterators, but
525 // replace it with a dummy instruction to remove uses.
526 //
527 // FIXME: We should not invert how this pass looks at operands to avoid
528 // this. Should track set of foldable movs instead of looking for uses
529 // when looking at a use.
530 Dst0.setReg(NewReg0);
531 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
532 MI->removeOperand(I);
533 MI->setDesc(TII->get(AMDGPU::IMPLICIT_DEF));
534
535 if (Fold.Commuted)
536 TII->commuteInstruction(*Inst32, false);
537 return true;
538 }
539
540 assert(!Fold.needsShrink() && "not handled");
541
542 if (Fold.isImm()) {
543 if (Old.isTied()) {
544 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode());
545 if (NewMFMAOpc == -1)
546 return false;
547 MI->setDesc(TII->get(NewMFMAOpc));
548 MI->untieRegOperand(0);
549 }
550 Old.ChangeToImmediate(Fold.ImmToFold);
551 return true;
552 }
553
554 if (Fold.isGlobal()) {
555 Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(),
556 Fold.OpToFold->getTargetFlags());
557 return true;
558 }
559
560 if (Fold.isFI()) {
561 Old.ChangeToFrameIndex(Fold.FrameIndexToFold);
562 return true;
563 }
564
565 MachineOperand *New = Fold.OpToFold;
566 Old.substVirtReg(New->getReg(), New->getSubReg(), *TRI);
567 Old.setIsUndef(New->isUndef());
568 return true;
569 }
570
571 static bool isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList,
572 const MachineInstr *MI) {
573 return any_of(FoldList, [&](const auto &C) { return C.UseMI == MI; });
574 }
575
576 static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
577 MachineInstr *MI, unsigned OpNo,
578 MachineOperand *FoldOp, bool Commuted = false,
579 int ShrinkOp = -1) {
580 // Skip additional folding on the same operand.
581 for (FoldCandidate &Fold : FoldList)
582 if (Fold.UseMI == MI && Fold.UseOpNo == OpNo)
583 return;
584 LLVM_DEBUG(dbgs() << "Append " << (Commuted ? "commuted" : "normal")
585 << " operand " << OpNo << "\n " << *MI);
586 FoldList.emplace_back(MI, OpNo, FoldOp, Commuted, ShrinkOp);
587 }
588
589 bool SIFoldOperandsImpl::tryAddToFoldList(
590 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
591 MachineOperand *OpToFold) const {
592 const unsigned Opc = MI->getOpcode();
593
594 auto tryToFoldAsFMAAKorMK = [&]() {
595 if (!OpToFold->isImm())
596 return false;
597
598 const bool TryAK = OpNo == 3;
599 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
600 MI->setDesc(TII->get(NewOpc));
601
602 // We have to fold into operand which would be Imm not into OpNo.
603 bool FoldAsFMAAKorMK =
604 tryAddToFoldList(FoldList, MI, TryAK ? 3 : 2, OpToFold);
605 if (FoldAsFMAAKorMK) {
606 // Untie Src2 of fmac.
607 MI->untieRegOperand(3);
608 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
609 if (OpNo == 1) {
610 MachineOperand &Op1 = MI->getOperand(1);
611 MachineOperand &Op2 = MI->getOperand(2);
612 Register OldReg = Op1.getReg();
613 // Operand 2 might be an inlinable constant
614 if (Op2.isImm()) {
615 Op1.ChangeToImmediate(Op2.getImm());
616 Op2.ChangeToRegister(OldReg, false);
617 } else {
618 Op1.setReg(Op2.getReg());
619 Op2.setReg(OldReg);
620 }
621 }
622 return true;
623 }
624 MI->setDesc(TII->get(Opc));
625 return false;
626 };
627
628 bool IsLegal = TII->isOperandLegal(*MI, OpNo, OpToFold);
629 if (!IsLegal && OpToFold->isImm()) {
630 FoldCandidate Fold(MI, OpNo, OpToFold);
631 IsLegal = canUseImmWithOpSel(Fold);
632 }
633
634 if (!IsLegal) {
635 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
636 unsigned NewOpc = macToMad(Opc);
637 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
638 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
639 // to fold the operand.
640 MI->setDesc(TII->get(NewOpc));
641 bool AddOpSel = !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::op_sel) &&
642 AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel);
643 if (AddOpSel)
644 MI->addOperand(MachineOperand::CreateImm(0));
645 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
646 if (FoldAsMAD) {
647 MI->untieRegOperand(OpNo);
648 return true;
649 }
650 if (AddOpSel)
651 MI->removeOperand(MI->getNumExplicitOperands() - 1);
652 MI->setDesc(TII->get(Opc));
653 }
654
655 // Special case for s_fmac_f32 if we are trying to fold into Src2.
656 // By transforming into fmaak we can untie Src2 and make folding legal.
657 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
658 if (tryToFoldAsFMAAKorMK())
659 return true;
660 }
661
662 // Special case for s_setreg_b32
663 if (OpToFold->isImm()) {
664 unsigned ImmOpc = 0;
665 if (Opc == AMDGPU::S_SETREG_B32)
666 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
667 else if (Opc == AMDGPU::S_SETREG_B32_mode)
668 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
669 if (ImmOpc) {
670 MI->setDesc(TII->get(ImmOpc));
671 appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
672 return true;
673 }
674 }
675
676 // If we are already folding into another operand of MI, then
677 // we can't commute the instruction, otherwise we risk making the
678 // other fold illegal.
679 if (isUseMIInFoldList(FoldList, MI))
680 return false;
681
682 // Operand is not legal, so try to commute the instruction to
683 // see if this makes it possible to fold.
684 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
685 bool CanCommute = TII->findCommutedOpIndices(*MI, OpNo, CommuteOpNo);
686 if (!CanCommute)
687 return false;
688
689 // One of operands might be an Imm operand, and OpNo may refer to it after
690 // the call of commuteInstruction() below. Such situations are avoided
691 // here explicitly as OpNo must be a register operand to be a candidate
692 // for memory folding.
693 if (!MI->getOperand(OpNo).isReg() || !MI->getOperand(CommuteOpNo).isReg())
694 return false;
695
696 if (!TII->commuteInstruction(*MI, false, OpNo, CommuteOpNo))
697 return false;
698
699 int Op32 = -1;
700 if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) {
701 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
702 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
703 (!OpToFold->isImm() && !OpToFold->isFI() && !OpToFold->isGlobal())) {
704 TII->commuteInstruction(*MI, false, OpNo, CommuteOpNo);
705 return false;
706 }
707
708 // Verify the other operand is a VGPR, otherwise we would violate the
709 // constant bus restriction.
710 MachineOperand &OtherOp = MI->getOperand(OpNo);
711 if (!OtherOp.isReg() ||
712 !TII->getRegisterInfo().isVGPR(*MRI, OtherOp.getReg()))
713 return false;
714
715 assert(MI->getOperand(1).isDef());
716
717 // Make sure to get the 32-bit version of the commuted opcode.
718 unsigned MaybeCommutedOpc = MI->getOpcode();
719 Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc);
720 }
721
722 appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32);
723 return true;
724 }
725
726 // Inlineable constant might have been folded into Imm operand of fmaak or
727 // fmamk and we are trying to fold a non-inlinable constant.
728 if ((Opc == AMDGPU::S_FMAAK_F32 || Opc == AMDGPU::S_FMAMK_F32) &&
729 !OpToFold->isReg() && !TII->isInlineConstant(*OpToFold)) {
730 unsigned ImmIdx = Opc == AMDGPU::S_FMAAK_F32 ? 3 : 2;
731 MachineOperand &OpImm = MI->getOperand(ImmIdx);
732 if (!OpImm.isReg() &&
733 TII->isInlineConstant(*MI, MI->getOperand(OpNo), OpImm))
734 return tryToFoldAsFMAAKorMK();
735 }
736
737 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
738 // By changing into fmamk we can untie Src2.
739 // If folding for Src0 happens first and it is identical operand to Src1 we
740 // should avoid transforming into fmamk which requires commuting as it would
741 // cause folding into Src1 to fail later on due to wrong OpNo used.
742 if (Opc == AMDGPU::S_FMAC_F32 &&
743 (OpNo != 1 || !MI->getOperand(1).isIdenticalTo(MI->getOperand(2)))) {
744 if (tryToFoldAsFMAAKorMK())
745 return true;
746 }
747
748 // Check the case where we might introduce a second constant operand to a
749 // scalar instruction
750 if (TII->isSALU(MI->getOpcode())) {
751 const MCInstrDesc &InstDesc = MI->getDesc();
752 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
753
754 // Fine if the operand can be encoded as an inline constant
755 if (!OpToFold->isReg() && !TII->isInlineConstant(*OpToFold, OpInfo)) {
756 // Otherwise check for another constant
757 for (unsigned i = 0, e = InstDesc.getNumOperands(); i != e; ++i) {
758 auto &Op = MI->getOperand(i);
759 if (OpNo != i && !Op.isReg() &&
760 !TII->isInlineConstant(Op, InstDesc.operands()[i]))
761 return false;
762 }
763 }
764 }
765
766 appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
767 return true;
768 }
769
770 bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
771 const MachineOperand &UseMO) const {
772 // Operands of SDWA instructions must be registers.
773 return !TII->isSDWA(MI);
774 }
775
776 // Find a def of the UseReg, check if it is a reg_sequence and find initializers
777 // for each subreg, tracking it to foldable inline immediate if possible.
778 // Returns true on success.
779 bool SIFoldOperandsImpl::getRegSeqInit(
780 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
781 Register UseReg, uint8_t OpTy) const {
782 MachineInstr *Def = MRI->getVRegDef(UseReg);
783 if (!Def || !Def->isRegSequence())
784 return false;
785
786 for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) {
787 MachineOperand *Sub = &Def->getOperand(I);
788 assert(Sub->isReg());
789
790 for (MachineInstr *SubDef = MRI->getVRegDef(Sub->getReg());
791 SubDef && Sub->isReg() && Sub->getReg().isVirtual() &&
792 !Sub->getSubReg() && TII->isFoldableCopy(*SubDef);
793 SubDef = MRI->getVRegDef(Sub->getReg())) {
794 MachineOperand *Op = &SubDef->getOperand(1);
795 if (Op->isImm()) {
796 if (TII->isInlineConstant(*Op, OpTy))
797 Sub = Op;
798 break;
799 }
800 if (!Op->isReg() || Op->getReg().isPhysical())
801 break;
802 Sub = Op;
803 }
804
805 Defs.emplace_back(Sub, Def->getOperand(I + 1).getImm());
806 }
807
808 return true;
809 }
810
811 bool SIFoldOperandsImpl::tryToFoldACImm(
812 const MachineOperand &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
813 SmallVectorImpl<FoldCandidate> &FoldList) const {
814 const MCInstrDesc &Desc = UseMI->getDesc();
815 if (UseOpIdx >= Desc.getNumOperands())
816 return false;
817
818 if (!AMDGPU::isSISrcInlinableOperand(Desc, UseOpIdx))
819 return false;
820
821 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
822 if (OpToFold.isImm() && TII->isInlineConstant(OpToFold, OpTy) &&
823 TII->isOperandLegal(*UseMI, UseOpIdx, &OpToFold)) {
824 UseMI->getOperand(UseOpIdx).ChangeToImmediate(OpToFold.getImm());
825 return true;
826 }
827
828 if (!OpToFold.isReg())
829 return false;
830
831 Register UseReg = OpToFold.getReg();
832 if (!UseReg.isVirtual())
833 return false;
834
835 if (isUseMIInFoldList(FoldList, UseMI))
836 return false;
837
838 // Maybe it is just a COPY of an immediate itself.
839 MachineInstr *Def = MRI->getVRegDef(UseReg);
840 MachineOperand &UseOp = UseMI->getOperand(UseOpIdx);
841 if (!UseOp.getSubReg() && Def && TII->isFoldableCopy(*Def)) {
842 MachineOperand &DefOp = Def->getOperand(1);
843 if (DefOp.isImm() && TII->isInlineConstant(DefOp, OpTy) &&
844 TII->isOperandLegal(*UseMI, UseOpIdx, &DefOp)) {
845 UseMI->getOperand(UseOpIdx).ChangeToImmediate(DefOp.getImm());
846 return true;
847 }
848 }
849
850 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
851 if (!getRegSeqInit(Defs, UseReg, OpTy))
852 return false;
853
854 int32_t Imm;
855 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
856 const MachineOperand *Op = Defs[I].first;
857 if (!Op->isImm())
858 return false;
859
860 auto SubImm = Op->getImm();
861 if (!I) {
862 Imm = SubImm;
863 if (!TII->isInlineConstant(*Op, OpTy) ||
864 !TII->isOperandLegal(*UseMI, UseOpIdx, Op))
865 return false;
866
867 continue;
868 }
869 if (Imm != SubImm)
870 return false; // Can only fold splat constants
871 }
872
873 appendFoldCandidate(FoldList, UseMI, UseOpIdx, Defs[0].first);
874 return true;
875 }
876
877 void SIFoldOperandsImpl::foldOperand(
878 MachineOperand &OpToFold, MachineInstr *UseMI, int UseOpIdx,
879 SmallVectorImpl<FoldCandidate> &FoldList,
880 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
881 const MachineOperand *UseOp = &UseMI->getOperand(UseOpIdx);
882
883 if (!isUseSafeToFold(*UseMI, *UseOp))
884 return;
885
886 // FIXME: Fold operands with subregs.
887 if (UseOp->isReg() && OpToFold.isReg() &&
888 (UseOp->isImplicit() || UseOp->getSubReg() != AMDGPU::NoSubRegister))
889 return;
890
891 // Special case for REG_SEQUENCE: We can't fold literals into
892 // REG_SEQUENCE instructions, so we have to fold them into the
893 // uses of REG_SEQUENCE.
894 if (UseMI->isRegSequence()) {
895 Register RegSeqDstReg = UseMI->getOperand(0).getReg();
896 unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm();
897
898 // Grab the use operands first
899 SmallVector<MachineOperand *, 4> UsesToProcess;
900 for (auto &Use : MRI->use_nodbg_operands(RegSeqDstReg))
901 UsesToProcess.push_back(&Use);
902 for (auto *RSUse : UsesToProcess) {
903 MachineInstr *RSUseMI = RSUse->getParent();
904
905 if (tryToFoldACImm(UseMI->getOperand(0), RSUseMI,
906 RSUseMI->getOperandNo(RSUse), FoldList))
907 continue;
908
909 if (RSUse->getSubReg() != RegSeqDstSubReg)
910 continue;
911
912 foldOperand(OpToFold, RSUseMI, RSUseMI->getOperandNo(RSUse), FoldList,
913 CopiesToReplace);
914 }
915 return;
916 }
917
918 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
919 return;
920
921 if (frameIndexMayFold(*UseMI, UseOpIdx, OpToFold)) {
922 // Verify that this is a stack access.
923 // FIXME: Should probably use stack pseudos before frame lowering.
924
925 if (TII->isMUBUF(*UseMI)) {
926 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() !=
927 MFI->getScratchRSrcReg())
928 return;
929
930 // Ensure this is either relative to the current frame or the current
931 // wave.
932 MachineOperand &SOff =
933 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset);
934 if (!SOff.isImm() || SOff.getImm() != 0)
935 return;
936 }
937
938 // A frame index will resolve to a positive constant, so it should always be
939 // safe to fold the addressing mode, even pre-GFX9.
940 UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex());
941
942 const unsigned Opc = UseMI->getOpcode();
943 if (TII->isFLATScratch(*UseMI) &&
944 AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) &&
945 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::saddr)) {
946 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opc);
947 UseMI->setDesc(TII->get(NewOpc));
948 }
949
950 return;
951 }
952
953 bool FoldingImmLike =
954 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
955
956 if (FoldingImmLike && UseMI->isCopy()) {
957 Register DestReg = UseMI->getOperand(0).getReg();
958 Register SrcReg = UseMI->getOperand(1).getReg();
959 assert(SrcReg.isVirtual());
960
961 const TargetRegisterClass *SrcRC = MRI->getRegClass(SrcReg);
962
963 // Don't fold into a copy to a physical register with the same class. Doing
964 // so would interfere with the register coalescer's logic which would avoid
965 // redundant initializations.
966 if (DestReg.isPhysical() && SrcRC->contains(DestReg))
967 return;
968
969 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(*MRI, DestReg);
970 if (!DestReg.isPhysical()) {
971 if (DestRC == &AMDGPU::AGPR_32RegClass &&
972 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
973 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64));
974 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
975 CopiesToReplace.push_back(UseMI);
976 return;
977 }
978 }
979
980 // In order to fold immediates into copies, we need to change the
981 // copy to a MOV.
982
983 unsigned MovOp = TII->getMovOpcode(DestRC);
984 if (MovOp == AMDGPU::COPY)
985 return;
986
987 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
988 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
989 while (ImpOpI != ImpOpE) {
990 MachineInstr::mop_iterator Tmp = ImpOpI;
991 ImpOpI++;
992 UseMI->removeOperand(UseMI->getOperandNo(Tmp));
993 }
994 UseMI->setDesc(TII->get(MovOp));
995
996 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
997 const auto &SrcOp = UseMI->getOperand(UseOpIdx);
998 MachineOperand NewSrcOp(SrcOp);
999 MachineFunction *MF = UseMI->getParent()->getParent();
1000 UseMI->removeOperand(1);
1001 UseMI->addOperand(*MF, MachineOperand::CreateImm(0)); // src0_modifiers
1002 UseMI->addOperand(NewSrcOp); // src0
1003 UseMI->addOperand(*MF, MachineOperand::CreateImm(0)); // op_sel
1004 UseOpIdx = 2;
1005 UseOp = &UseMI->getOperand(UseOpIdx);
1006 }
1007 CopiesToReplace.push_back(UseMI);
1008 } else {
1009 if (UseMI->isCopy() && OpToFold.isReg() &&
1010 UseMI->getOperand(0).getReg().isVirtual() &&
1011 !UseMI->getOperand(1).getSubReg()) {
1012 LLVM_DEBUG(dbgs() << "Folding " << OpToFold << "\n into " << *UseMI);
1013 unsigned Size = TII->getOpSize(*UseMI, 1);
1014 Register UseReg = OpToFold.getReg();
1015 UseMI->getOperand(1).setReg(UseReg);
1016 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
1017 UseMI->getOperand(1).setIsKill(false);
1018 CopiesToReplace.push_back(UseMI);
1019 OpToFold.setIsKill(false);
1020
1021 // Remove kill flags as kills may now be out of order with uses.
1022 MRI->clearKillFlags(OpToFold.getReg());
1023
1024 // That is very tricky to store a value into an AGPR. v_accvgpr_write_b32
1025 // can only accept VGPR or inline immediate. Recreate a reg_sequence with
1026 // its initializers right here, so we will rematerialize immediates and
1027 // avoid copies via different reg classes.
1028 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
1029 if (Size > 4 && TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) &&
1030 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
1031 const DebugLoc &DL = UseMI->getDebugLoc();
1032 MachineBasicBlock &MBB = *UseMI->getParent();
1033
1034 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE));
1035 for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I)
1036 UseMI->removeOperand(I);
1037
1038 MachineInstrBuilder B(*MBB.getParent(), UseMI);
1039 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
1040 SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs;
1041 for (unsigned I = 0; I < Size / 4; ++I) {
1042 MachineOperand *Def = Defs[I].first;
1043 TargetInstrInfo::RegSubRegPair CopyToVGPR;
1044 if (Def->isImm() &&
1045 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
1046 int64_t Imm = Def->getImm();
1047
1048 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
1049 BuildMI(MBB, UseMI, DL,
1050 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm);
1051 B.addReg(Tmp);
1052 } else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) {
1053 auto Src = getRegSubRegPair(*Def);
1054 Def->setIsKill(false);
1055 if (!SeenAGPRs.insert(Src)) {
1056 // We cannot build a reg_sequence out of the same registers, they
1057 // must be copied. Better do it here before copyPhysReg() created
1058 // several reads to do the AGPR->VGPR->AGPR copy.
1059 CopyToVGPR = Src;
1060 } else {
1061 B.addReg(Src.Reg, Def->isUndef() ? RegState::Undef : 0,
1062 Src.SubReg);
1063 }
1064 } else {
1065 assert(Def->isReg());
1066 Def->setIsKill(false);
1067 auto Src = getRegSubRegPair(*Def);
1068
1069 // Direct copy from SGPR to AGPR is not possible. To avoid creation
1070 // of exploded copies SGPR->VGPR->AGPR in the copyPhysReg() later,
1071 // create a copy here and track if we already have such a copy.
1072 if (TRI->isSGPRReg(*MRI, Src.Reg)) {
1073 CopyToVGPR = Src;
1074 } else {
1075 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
1076 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def);
1077 B.addReg(Tmp);
1078 }
1079 }
1080
1081 if (CopyToVGPR.Reg) {
1082 Register Vgpr;
1083 if (VGPRCopies.count(CopyToVGPR)) {
1084 Vgpr = VGPRCopies[CopyToVGPR];
1085 } else {
1086 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1087 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def);
1088 VGPRCopies[CopyToVGPR] = Vgpr;
1089 }
1090 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
1091 BuildMI(MBB, UseMI, DL,
1092 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr);
1093 B.addReg(Tmp);
1094 }
1095
1096 B.addImm(Defs[I].second);
1097 }
1098 LLVM_DEBUG(dbgs() << "Folded " << *UseMI);
1099 return;
1100 }
1101
1102 if (Size != 4)
1103 return;
1104
1105 Register Reg0 = UseMI->getOperand(0).getReg();
1106 Register Reg1 = UseMI->getOperand(1).getReg();
1107 if (TRI->isAGPR(*MRI, Reg0) && TRI->isVGPR(*MRI, Reg1))
1108 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64));
1109 else if (TRI->isVGPR(*MRI, Reg0) && TRI->isAGPR(*MRI, Reg1))
1110 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64));
1111 else if (ST->hasGFX90AInsts() && TRI->isAGPR(*MRI, Reg0) &&
1112 TRI->isAGPR(*MRI, Reg1))
1113 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32));
1114 return;
1115 }
1116
1117 unsigned UseOpc = UseMI->getOpcode();
1118 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1119 (UseOpc == AMDGPU::V_READLANE_B32 &&
1120 (int)UseOpIdx ==
1121 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) {
1122 // %vgpr = V_MOV_B32 imm
1123 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1124 // =>
1125 // %sgpr = S_MOV_B32 imm
1126 if (FoldingImmLike) {
1127 if (execMayBeModifiedBeforeUse(*MRI,
1128 UseMI->getOperand(UseOpIdx).getReg(),
1129 *OpToFold.getParent(),
1130 *UseMI))
1131 return;
1132
1133 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32));
1134
1135 if (OpToFold.isImm())
1136 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
1137 else
1138 UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getIndex());
1139 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
1140 return;
1141 }
1142
1143 if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) {
1144 if (execMayBeModifiedBeforeUse(*MRI,
1145 UseMI->getOperand(UseOpIdx).getReg(),
1146 *OpToFold.getParent(),
1147 *UseMI))
1148 return;
1149
1150 // %vgpr = COPY %sgpr0
1151 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1152 // =>
1153 // %sgpr1 = COPY %sgpr0
1154 UseMI->setDesc(TII->get(AMDGPU::COPY));
1155 UseMI->getOperand(1).setReg(OpToFold.getReg());
1156 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
1157 UseMI->getOperand(1).setIsKill(false);
1158 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
1159 return;
1160 }
1161 }
1162
1163 const MCInstrDesc &UseDesc = UseMI->getDesc();
1164
1165 // Don't fold into target independent nodes. Target independent opcodes
1166 // don't have defined register classes.
1167 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1168 UseDesc.operands()[UseOpIdx].RegClass == -1)
1169 return;
1170 }
1171
1172 if (!FoldingImmLike) {
1173 if (OpToFold.isReg() && ST->needsAlignedVGPRs()) {
1174 // Don't fold if OpToFold doesn't hold an aligned register.
1175 const TargetRegisterClass *RC =
1176 TRI->getRegClassForReg(*MRI, OpToFold.getReg());
1177 assert(RC);
1178 if (TRI->hasVectorRegisters(RC) && OpToFold.getSubReg()) {
1179 unsigned SubReg = OpToFold.getSubReg();
1180 if (const TargetRegisterClass *SubRC =
1181 TRI->getSubRegisterClass(RC, SubReg))
1182 RC = SubRC;
1183 }
1184
1185 if (!RC || !TRI->isProperlyAlignedRC(*RC))
1186 return;
1187 }
1188
1189 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
1190
1191 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1192 // to enable more folding opportunities. The shrink operands pass
1193 // already does this.
1194 return;
1195 }
1196
1197
1198 const MCInstrDesc &FoldDesc = OpToFold.getParent()->getDesc();
1199 const TargetRegisterClass *FoldRC =
1200 TRI->getRegClass(FoldDesc.operands()[0].RegClass);
1201
1202 // Split 64-bit constants into 32-bits for folding.
1203 if (UseOp->getSubReg() && AMDGPU::getRegBitWidth(*FoldRC) == 64) {
1204 Register UseReg = UseOp->getReg();
1205 const TargetRegisterClass *UseRC = MRI->getRegClass(UseReg);
1206 if (AMDGPU::getRegBitWidth(*UseRC) != 64)
1207 return;
1208
1209 APInt Imm(64, OpToFold.getImm());
1210 if (UseOp->getSubReg() == AMDGPU::sub0) {
1211 Imm = Imm.getLoBits(32);
1212 } else {
1213 assert(UseOp->getSubReg() == AMDGPU::sub1);
1214 Imm = Imm.getHiBits(32);
1215 }
1216
1217 MachineOperand ImmOp = MachineOperand::CreateImm(Imm.getSExtValue());
1218 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &ImmOp);
1219 return;
1220 }
1221
1222 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
1223 }
1224
1225 static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1226 uint32_t LHS, uint32_t RHS) {
1227 switch (Opcode) {
1228 case AMDGPU::V_AND_B32_e64:
1229 case AMDGPU::V_AND_B32_e32:
1230 case AMDGPU::S_AND_B32:
1231 Result = LHS & RHS;
1232 return true;
1233 case AMDGPU::V_OR_B32_e64:
1234 case AMDGPU::V_OR_B32_e32:
1235 case AMDGPU::S_OR_B32:
1236 Result = LHS | RHS;
1237 return true;
1238 case AMDGPU::V_XOR_B32_e64:
1239 case AMDGPU::V_XOR_B32_e32:
1240 case AMDGPU::S_XOR_B32:
1241 Result = LHS ^ RHS;
1242 return true;
1243 case AMDGPU::S_XNOR_B32:
1244 Result = ~(LHS ^ RHS);
1245 return true;
1246 case AMDGPU::S_NAND_B32:
1247 Result = ~(LHS & RHS);
1248 return true;
1249 case AMDGPU::S_NOR_B32:
1250 Result = ~(LHS | RHS);
1251 return true;
1252 case AMDGPU::S_ANDN2_B32:
1253 Result = LHS & ~RHS;
1254 return true;
1255 case AMDGPU::S_ORN2_B32:
1256 Result = LHS | ~RHS;
1257 return true;
1258 case AMDGPU::V_LSHL_B32_e64:
1259 case AMDGPU::V_LSHL_B32_e32:
1260 case AMDGPU::S_LSHL_B32:
1261 // The instruction ignores the high bits for out of bounds shifts.
1262 Result = LHS << (RHS & 31);
1263 return true;
1264 case AMDGPU::V_LSHLREV_B32_e64:
1265 case AMDGPU::V_LSHLREV_B32_e32:
1266 Result = RHS << (LHS & 31);
1267 return true;
1268 case AMDGPU::V_LSHR_B32_e64:
1269 case AMDGPU::V_LSHR_B32_e32:
1270 case AMDGPU::S_LSHR_B32:
1271 Result = LHS >> (RHS & 31);
1272 return true;
1273 case AMDGPU::V_LSHRREV_B32_e64:
1274 case AMDGPU::V_LSHRREV_B32_e32:
1275 Result = RHS >> (LHS & 31);
1276 return true;
1277 case AMDGPU::V_ASHR_I32_e64:
1278 case AMDGPU::V_ASHR_I32_e32:
1279 case AMDGPU::S_ASHR_I32:
1280 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1281 return true;
1282 case AMDGPU::V_ASHRREV_I32_e64:
1283 case AMDGPU::V_ASHRREV_I32_e32:
1284 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1285 return true;
1286 default:
1287 return false;
1288 }
1289 }
1290
1291 static unsigned getMovOpc(bool IsScalar) {
1292 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1293 }
1294
1295 static void mutateCopyOp(MachineInstr &MI, const MCInstrDesc &NewDesc) {
1296 MI.setDesc(NewDesc);
1297
1298 // Remove any leftover implicit operands from mutating the instruction. e.g.
1299 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
1300 // anymore.
1301 const MCInstrDesc &Desc = MI.getDesc();
1302 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
1303 Desc.implicit_defs().size();
1304
1305 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
1306 MI.removeOperand(I);
1307 }
1308
1309 MachineOperand *
1310 SIFoldOperandsImpl::getImmOrMaterializedImm(MachineOperand &Op) const {
1311 // If this has a subregister, it obviously is a register source.
1312 if (!Op.isReg() || Op.getSubReg() != AMDGPU::NoSubRegister ||
1313 !Op.getReg().isVirtual())
1314 return &Op;
1315
1316 MachineInstr *Def = MRI->getVRegDef(Op.getReg());
1317 if (Def && Def->isMoveImmediate()) {
1318 MachineOperand &ImmSrc = Def->getOperand(1);
1319 if (ImmSrc.isImm())
1320 return &ImmSrc;
1321 }
1322
1323 return &Op;
1324 }
1325
1326 // Try to simplify operations with a constant that may appear after instruction
1327 // selection.
1328 // TODO: See if a frame index with a fixed offset can fold.
1329 bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1330 if (!MI->allImplicitDefsAreDead())
1331 return false;
1332
1333 unsigned Opc = MI->getOpcode();
1334
1335 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1336 if (Src0Idx == -1)
1337 return false;
1338 MachineOperand *Src0 = getImmOrMaterializedImm(MI->getOperand(Src0Idx));
1339
1340 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1341 Opc == AMDGPU::S_NOT_B32) &&
1342 Src0->isImm()) {
1343 MI->getOperand(1).ChangeToImmediate(~Src0->getImm());
1344 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32)));
1345 return true;
1346 }
1347
1348 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1349 if (Src1Idx == -1)
1350 return false;
1351 MachineOperand *Src1 = getImmOrMaterializedImm(MI->getOperand(Src1Idx));
1352
1353 if (!Src0->isImm() && !Src1->isImm())
1354 return false;
1355
1356 // and k0, k1 -> v_mov_b32 (k0 & k1)
1357 // or k0, k1 -> v_mov_b32 (k0 | k1)
1358 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1359 if (Src0->isImm() && Src1->isImm()) {
1360 int32_t NewImm;
1361 if (!evalBinaryInstruction(Opc, NewImm, Src0->getImm(), Src1->getImm()))
1362 return false;
1363
1364 bool IsSGPR = TRI->isSGPRReg(*MRI, MI->getOperand(0).getReg());
1365
1366 // Be careful to change the right operand, src0 may belong to a different
1367 // instruction.
1368 MI->getOperand(Src0Idx).ChangeToImmediate(NewImm);
1369 MI->removeOperand(Src1Idx);
1370 mutateCopyOp(*MI, TII->get(getMovOpc(IsSGPR)));
1371 return true;
1372 }
1373
1374 if (!MI->isCommutable())
1375 return false;
1376
1377 if (Src0->isImm() && !Src1->isImm()) {
1378 std::swap(Src0, Src1);
1379 std::swap(Src0Idx, Src1Idx);
1380 }
1381
1382 int32_t Src1Val = static_cast<int32_t>(Src1->getImm());
1383 if (Opc == AMDGPU::V_OR_B32_e64 ||
1384 Opc == AMDGPU::V_OR_B32_e32 ||
1385 Opc == AMDGPU::S_OR_B32) {
1386 if (Src1Val == 0) {
1387 // y = or x, 0 => y = copy x
1388 MI->removeOperand(Src1Idx);
1389 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1390 } else if (Src1Val == -1) {
1391 // y = or x, -1 => y = v_mov_b32 -1
1392 MI->removeOperand(Src1Idx);
1393 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32)));
1394 } else
1395 return false;
1396
1397 return true;
1398 }
1399
1400 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1401 Opc == AMDGPU::S_AND_B32) {
1402 if (Src1Val == 0) {
1403 // y = and x, 0 => y = v_mov_b32 0
1404 MI->removeOperand(Src0Idx);
1405 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32)));
1406 } else if (Src1Val == -1) {
1407 // y = and x, -1 => y = copy x
1408 MI->removeOperand(Src1Idx);
1409 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1410 } else
1411 return false;
1412
1413 return true;
1414 }
1415
1416 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1417 Opc == AMDGPU::S_XOR_B32) {
1418 if (Src1Val == 0) {
1419 // y = xor x, 0 => y = copy x
1420 MI->removeOperand(Src1Idx);
1421 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1422 return true;
1423 }
1424 }
1425
1426 return false;
1427 }
1428
1429 // Try to fold an instruction into a simpler one
1430 bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1431 unsigned Opc = MI.getOpcode();
1432 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1433 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1434 return false;
1435
1436 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1437 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1438 if (!Src1->isIdenticalTo(*Src0)) {
1439 auto *Src0Imm = getImmOrMaterializedImm(*Src0);
1440 auto *Src1Imm = getImmOrMaterializedImm(*Src1);
1441 if (!Src1Imm->isIdenticalTo(*Src0Imm))
1442 return false;
1443 }
1444
1445 int Src1ModIdx =
1446 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers);
1447 int Src0ModIdx =
1448 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
1449 if ((Src1ModIdx != -1 && MI.getOperand(Src1ModIdx).getImm() != 0) ||
1450 (Src0ModIdx != -1 && MI.getOperand(Src0ModIdx).getImm() != 0))
1451 return false;
1452
1453 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1454 auto &NewDesc =
1455 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false));
1456 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1457 if (Src2Idx != -1)
1458 MI.removeOperand(Src2Idx);
1459 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1));
1460 if (Src1ModIdx != -1)
1461 MI.removeOperand(Src1ModIdx);
1462 if (Src0ModIdx != -1)
1463 MI.removeOperand(Src0ModIdx);
1464 mutateCopyOp(MI, NewDesc);
1465 LLVM_DEBUG(dbgs() << MI);
1466 return true;
1467 }
1468
1469 bool SIFoldOperandsImpl::tryFoldZeroHighBits(MachineInstr &MI) const {
1470 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 &&
1471 MI.getOpcode() != AMDGPU::V_AND_B32_e32)
1472 return false;
1473
1474 MachineOperand *Src0 = getImmOrMaterializedImm(MI.getOperand(1));
1475 if (!Src0->isImm() || Src0->getImm() != 0xffff)
1476 return false;
1477
1478 Register Src1 = MI.getOperand(2).getReg();
1479 MachineInstr *SrcDef = MRI->getVRegDef(Src1);
1480 if (!ST->zeroesHigh16BitsOfDest(SrcDef->getOpcode()))
1481 return false;
1482
1483 Register Dst = MI.getOperand(0).getReg();
1484 MRI->replaceRegWith(Dst, Src1);
1485 if (!MI.getOperand(2).isKill())
1486 MRI->clearKillFlags(Src1);
1487 MI.eraseFromParent();
1488 return true;
1489 }
1490
1491 bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
1492 MachineOperand &OpToFold) const {
1493 // We need mutate the operands of new mov instructions to add implicit
1494 // uses of EXEC, but adding them invalidates the use_iterator, so defer
1495 // this.
1496 SmallVector<MachineInstr *, 4> CopiesToReplace;
1497 SmallVector<FoldCandidate, 4> FoldList;
1498 MachineOperand &Dst = MI.getOperand(0);
1499 bool Changed = false;
1500
1501 if (OpToFold.isImm()) {
1502 for (auto &UseMI :
1503 make_early_inc_range(MRI->use_nodbg_instructions(Dst.getReg()))) {
1504 // Folding the immediate may reveal operations that can be constant
1505 // folded or replaced with a copy. This can happen for example after
1506 // frame indices are lowered to constants or from splitting 64-bit
1507 // constants.
1508 //
1509 // We may also encounter cases where one or both operands are
1510 // immediates materialized into a register, which would ordinarily not
1511 // be folded due to multiple uses or operand constraints.
1512 if (tryConstantFoldOp(&UseMI)) {
1513 LLVM_DEBUG(dbgs() << "Constant folded " << UseMI);
1514 Changed = true;
1515 }
1516 }
1517 }
1518
1519 SmallVector<MachineOperand *, 4> UsesToProcess;
1520 for (auto &Use : MRI->use_nodbg_operands(Dst.getReg()))
1521 UsesToProcess.push_back(&Use);
1522 for (auto *U : UsesToProcess) {
1523 MachineInstr *UseMI = U->getParent();
1524 foldOperand(OpToFold, UseMI, UseMI->getOperandNo(U), FoldList,
1525 CopiesToReplace);
1526 }
1527
1528 if (CopiesToReplace.empty() && FoldList.empty())
1529 return Changed;
1530
1531 MachineFunction *MF = MI.getParent()->getParent();
1532 // Make sure we add EXEC uses to any new v_mov instructions created.
1533 for (MachineInstr *Copy : CopiesToReplace)
1534 Copy->addImplicitDefUseOperands(*MF);
1535
1536 for (FoldCandidate &Fold : FoldList) {
1537 assert(!Fold.isReg() || Fold.OpToFold);
1538 if (Fold.isReg() && Fold.OpToFold->getReg().isVirtual()) {
1539 Register Reg = Fold.OpToFold->getReg();
1540 MachineInstr *DefMI = Fold.OpToFold->getParent();
1541 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) &&
1542 execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI))
1543 continue;
1544 }
1545 if (updateOperand(Fold)) {
1546 // Clear kill flags.
1547 if (Fold.isReg()) {
1548 assert(Fold.OpToFold && Fold.OpToFold->isReg());
1549 // FIXME: Probably shouldn't bother trying to fold if not an
1550 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
1551 // copies.
1552 MRI->clearKillFlags(Fold.OpToFold->getReg());
1553 }
1554 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
1555 << static_cast<int>(Fold.UseOpNo) << " of "
1556 << *Fold.UseMI);
1557 } else if (Fold.Commuted) {
1558 // Restoring instruction's original operand order if fold has failed.
1559 TII->commuteInstruction(*Fold.UseMI, false);
1560 }
1561 }
1562 return true;
1563 }
1564
1565 bool SIFoldOperandsImpl::tryFoldFoldableCopy(
1566 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
1567 Register DstReg = MI.getOperand(0).getReg();
1568 // Specially track simple redefs of m0 to the same value in a block, so we
1569 // can erase the later ones.
1570 if (DstReg == AMDGPU::M0) {
1571 MachineOperand &NewM0Val = MI.getOperand(1);
1572 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(NewM0Val)) {
1573 MI.eraseFromParent();
1574 return true;
1575 }
1576
1577 // We aren't tracking other physical registers
1578 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
1579 ? nullptr
1580 : &NewM0Val;
1581 return false;
1582 }
1583
1584 MachineOperand *OpToFoldPtr;
1585 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
1586 // Folding when any src_modifiers are non-zero is unsupported
1587 if (TII->hasAnyModifiersSet(MI))
1588 return false;
1589 OpToFoldPtr = &MI.getOperand(2);
1590 } else
1591 OpToFoldPtr = &MI.getOperand(1);
1592 MachineOperand &OpToFold = *OpToFoldPtr;
1593 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1594
1595 // FIXME: We could also be folding things like TargetIndexes.
1596 if (!FoldingImm && !OpToFold.isReg())
1597 return false;
1598
1599 if (OpToFold.isReg() && !OpToFold.getReg().isVirtual())
1600 return false;
1601
1602 // Prevent folding operands backwards in the function. For example,
1603 // the COPY opcode must not be replaced by 1 in this example:
1604 //
1605 // %3 = COPY %vgpr0; VGPR_32:%3
1606 // ...
1607 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
1608 if (!DstReg.isVirtual())
1609 return false;
1610
1611 if (OpToFold.isReg() &&
1612 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, OpToFold.getReg(), MI))
1613 return true;
1614
1615 bool Changed = foldInstOperand(MI, OpToFold);
1616
1617 // If we managed to fold all uses of this copy then we might as well
1618 // delete it now.
1619 // The only reason we need to follow chains of copies here is that
1620 // tryFoldRegSequence looks forward through copies before folding a
1621 // REG_SEQUENCE into its eventual users.
1622 auto *InstToErase = &MI;
1623 while (MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) {
1624 auto &SrcOp = InstToErase->getOperand(1);
1625 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
1626 InstToErase->eraseFromParent();
1627 Changed = true;
1628 InstToErase = nullptr;
1629 if (!SrcReg || SrcReg.isPhysical())
1630 break;
1631 InstToErase = MRI->getVRegDef(SrcReg);
1632 if (!InstToErase || !TII->isFoldableCopy(*InstToErase))
1633 break;
1634 }
1635
1636 if (InstToErase && InstToErase->isRegSequence() &&
1637 MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) {
1638 InstToErase->eraseFromParent();
1639 Changed = true;
1640 }
1641
1642 return Changed;
1643 }
1644
1645 // Clamp patterns are canonically selected to v_max_* instructions, so only
1646 // handle them.
1647 const MachineOperand *
1648 SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
1649 unsigned Op = MI.getOpcode();
1650 switch (Op) {
1651 case AMDGPU::V_MAX_F32_e64:
1652 case AMDGPU::V_MAX_F16_e64:
1653 case AMDGPU::V_MAX_F16_t16_e64:
1654 case AMDGPU::V_MAX_F16_fake16_e64:
1655 case AMDGPU::V_MAX_F64_e64:
1656 case AMDGPU::V_MAX_NUM_F64_e64:
1657 case AMDGPU::V_PK_MAX_F16: {
1658 if (MI.mayRaiseFPException())
1659 return nullptr;
1660
1661 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
1662 return nullptr;
1663
1664 // Make sure sources are identical.
1665 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1666 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1667 if (!Src0->isReg() || !Src1->isReg() ||
1668 Src0->getReg() != Src1->getReg() ||
1669 Src0->getSubReg() != Src1->getSubReg() ||
1670 Src0->getSubReg() != AMDGPU::NoSubRegister)
1671 return nullptr;
1672
1673 // Can't fold up if we have modifiers.
1674 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1675 return nullptr;
1676
1677 unsigned Src0Mods
1678 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm();
1679 unsigned Src1Mods
1680 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
1681
1682 // Having a 0 op_sel_hi would require swizzling the output in the source
1683 // instruction, which we can't do.
1684 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1
1685 : 0u;
1686 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
1687 return nullptr;
1688 return Src0;
1689 }
1690 default:
1691 return nullptr;
1692 }
1693 }
1694
1695 // FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
1696 bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
1697 const MachineOperand *ClampSrc = isClamp(MI);
1698 if (!ClampSrc || !MRI->hasOneNonDBGUser(ClampSrc->getReg()))
1699 return false;
1700
1701 MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg());
1702
1703 // The type of clamp must be compatible.
1704 if (TII->getClampMask(*Def) != TII->getClampMask(MI))
1705 return false;
1706
1707 if (Def->mayRaiseFPException())
1708 return false;
1709
1710 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
1711 if (!DefClamp)
1712 return false;
1713
1714 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
1715
1716 // Clamp is applied after omod, so it is OK if omod is set.
1717 DefClamp->setImm(1);
1718
1719 Register DefReg = Def->getOperand(0).getReg();
1720 Register MIDstReg = MI.getOperand(0).getReg();
1721 if (TRI->isSGPRReg(*MRI, DefReg)) {
1722 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
1723 // instruction with a VGPR dst.
1724 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
1725 MIDstReg)
1726 .addReg(DefReg);
1727 } else {
1728 MRI->replaceRegWith(MIDstReg, DefReg);
1729 }
1730 MI.eraseFromParent();
1731
1732 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
1733 // instruction, so we might as well convert it to the more flexible VOP3-only
1734 // mad/fma form.
1735 if (TII->convertToThreeAddress(*Def, nullptr, nullptr))
1736 Def->eraseFromParent();
1737
1738 return true;
1739 }
1740
1741 static int getOModValue(unsigned Opc, int64_t Val) {
1742 switch (Opc) {
1743 case AMDGPU::V_MUL_F64_e64:
1744 case AMDGPU::V_MUL_F64_pseudo_e64: {
1745 switch (Val) {
1746 case 0x3fe0000000000000: // 0.5
1747 return SIOutMods::DIV2;
1748 case 0x4000000000000000: // 2.0
1749 return SIOutMods::MUL2;
1750 case 0x4010000000000000: // 4.0
1751 return SIOutMods::MUL4;
1752 default:
1753 return SIOutMods::NONE;
1754 }
1755 }
1756 case AMDGPU::V_MUL_F32_e64: {
1757 switch (static_cast<uint32_t>(Val)) {
1758 case 0x3f000000: // 0.5
1759 return SIOutMods::DIV2;
1760 case 0x40000000: // 2.0
1761 return SIOutMods::MUL2;
1762 case 0x40800000: // 4.0
1763 return SIOutMods::MUL4;
1764 default:
1765 return SIOutMods::NONE;
1766 }
1767 }
1768 case AMDGPU::V_MUL_F16_e64:
1769 case AMDGPU::V_MUL_F16_t16_e64:
1770 case AMDGPU::V_MUL_F16_fake16_e64: {
1771 switch (static_cast<uint16_t>(Val)) {
1772 case 0x3800: // 0.5
1773 return SIOutMods::DIV2;
1774 case 0x4000: // 2.0
1775 return SIOutMods::MUL2;
1776 case 0x4400: // 4.0
1777 return SIOutMods::MUL4;
1778 default:
1779 return SIOutMods::NONE;
1780 }
1781 }
1782 default:
1783 llvm_unreachable("invalid mul opcode");
1784 }
1785 }
1786
1787 // FIXME: Does this really not support denormals with f16?
1788 // FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
1789 // handled, so will anything other than that break?
1790 std::pair<const MachineOperand *, int>
1791 SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
1792 unsigned Op = MI.getOpcode();
1793 switch (Op) {
1794 case AMDGPU::V_MUL_F64_e64:
1795 case AMDGPU::V_MUL_F64_pseudo_e64:
1796 case AMDGPU::V_MUL_F32_e64:
1797 case AMDGPU::V_MUL_F16_t16_e64:
1798 case AMDGPU::V_MUL_F16_fake16_e64:
1799 case AMDGPU::V_MUL_F16_e64: {
1800 // If output denormals are enabled, omod is ignored.
1801 if ((Op == AMDGPU::V_MUL_F32_e64 &&
1802 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
1803 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
1804 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
1805 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
1806 MFI->getMode().FP64FP16Denormals.Output !=
1807 DenormalMode::PreserveSign) ||
1808 MI.mayRaiseFPException())
1809 return std::pair(nullptr, SIOutMods::NONE);
1810
1811 const MachineOperand *RegOp = nullptr;
1812 const MachineOperand *ImmOp = nullptr;
1813 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1814 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1815 if (Src0->isImm()) {
1816 ImmOp = Src0;
1817 RegOp = Src1;
1818 } else if (Src1->isImm()) {
1819 ImmOp = Src1;
1820 RegOp = Src0;
1821 } else
1822 return std::pair(nullptr, SIOutMods::NONE);
1823
1824 int OMod = getOModValue(Op, ImmOp->getImm());
1825 if (OMod == SIOutMods::NONE ||
1826 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
1827 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
1828 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
1829 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
1830 return std::pair(nullptr, SIOutMods::NONE);
1831
1832 return std::pair(RegOp, OMod);
1833 }
1834 case AMDGPU::V_ADD_F64_e64:
1835 case AMDGPU::V_ADD_F64_pseudo_e64:
1836 case AMDGPU::V_ADD_F32_e64:
1837 case AMDGPU::V_ADD_F16_e64:
1838 case AMDGPU::V_ADD_F16_t16_e64:
1839 case AMDGPU::V_ADD_F16_fake16_e64: {
1840 // If output denormals are enabled, omod is ignored.
1841 if ((Op == AMDGPU::V_ADD_F32_e64 &&
1842 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
1843 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
1844 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
1845 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
1846 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
1847 return std::pair(nullptr, SIOutMods::NONE);
1848
1849 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
1850 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1851 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1852
1853 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
1854 Src0->getSubReg() == Src1->getSubReg() &&
1855 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) &&
1856 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) &&
1857 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) &&
1858 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1859 return std::pair(Src0, SIOutMods::MUL2);
1860
1861 return std::pair(nullptr, SIOutMods::NONE);
1862 }
1863 default:
1864 return std::pair(nullptr, SIOutMods::NONE);
1865 }
1866 }
1867
1868 // FIXME: Does this need to check IEEE bit on function?
1869 bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
1870 const MachineOperand *RegOp;
1871 int OMod;
1872 std::tie(RegOp, OMod) = isOMod(MI);
1873 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
1874 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
1875 !MRI->hasOneNonDBGUser(RegOp->getReg()))
1876 return false;
1877
1878 MachineInstr *Def = MRI->getVRegDef(RegOp->getReg());
1879 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod);
1880 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
1881 return false;
1882
1883 if (Def->mayRaiseFPException())
1884 return false;
1885
1886 // Clamp is applied after omod. If the source already has clamp set, don't
1887 // fold it.
1888 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp))
1889 return false;
1890
1891 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
1892
1893 DefOMod->setImm(OMod);
1894 MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
1895 // Kill flags can be wrong if we replaced a def inside a loop with a def
1896 // outside the loop.
1897 MRI->clearKillFlags(Def->getOperand(0).getReg());
1898 MI.eraseFromParent();
1899
1900 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
1901 // instruction, so we might as well convert it to the more flexible VOP3-only
1902 // mad/fma form.
1903 if (TII->convertToThreeAddress(*Def, nullptr, nullptr))
1904 Def->eraseFromParent();
1905
1906 return true;
1907 }
1908
1909 // Try to fold a reg_sequence with vgpr output and agpr inputs into an
1910 // instruction which can take an agpr. So far that means a store.
1911 bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
1912 assert(MI.isRegSequence());
1913 auto Reg = MI.getOperand(0).getReg();
1914
1915 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(*MRI, Reg) ||
1916 !MRI->hasOneNonDBGUse(Reg))
1917 return false;
1918
1919 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
1920 if (!getRegSeqInit(Defs, Reg, MCOI::OPERAND_REGISTER))
1921 return false;
1922
1923 for (auto &[Op, SubIdx] : Defs) {
1924 if (!Op->isReg())
1925 return false;
1926 if (TRI->isAGPR(*MRI, Op->getReg()))
1927 continue;
1928 // Maybe this is a COPY from AREG
1929 const MachineInstr *SubDef = MRI->getVRegDef(Op->getReg());
1930 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(1).getSubReg())
1931 return false;
1932 if (!TRI->isAGPR(*MRI, SubDef->getOperand(1).getReg()))
1933 return false;
1934 }
1935
1936 MachineOperand *Op = &*MRI->use_nodbg_begin(Reg);
1937 MachineInstr *UseMI = Op->getParent();
1938 while (UseMI->isCopy() && !Op->getSubReg()) {
1939 Reg = UseMI->getOperand(0).getReg();
1940 if (!TRI->isVGPR(*MRI, Reg) || !MRI->hasOneNonDBGUse(Reg))
1941 return false;
1942 Op = &*MRI->use_nodbg_begin(Reg);
1943 UseMI = Op->getParent();
1944 }
1945
1946 if (Op->getSubReg())
1947 return false;
1948
1949 unsigned OpIdx = Op - &UseMI->getOperand(0);
1950 const MCInstrDesc &InstDesc = UseMI->getDesc();
1951 const TargetRegisterClass *OpRC =
1952 TII->getRegClass(InstDesc, OpIdx, TRI, *MI.getMF());
1953 if (!OpRC || !TRI->isVectorSuperClass(OpRC))
1954 return false;
1955
1956 const auto *NewDstRC = TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg));
1957 auto Dst = MRI->createVirtualRegister(NewDstRC);
1958 auto RS = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1959 TII->get(AMDGPU::REG_SEQUENCE), Dst);
1960
1961 for (auto &[Def, SubIdx] : Defs) {
1962 Def->setIsKill(false);
1963 if (TRI->isAGPR(*MRI, Def->getReg())) {
1964 RS.add(*Def);
1965 } else { // This is a copy
1966 MachineInstr *SubDef = MRI->getVRegDef(Def->getReg());
1967 SubDef->getOperand(1).setIsKill(false);
1968 RS.addReg(SubDef->getOperand(1).getReg(), 0, Def->getSubReg());
1969 }
1970 RS.addImm(SubIdx);
1971 }
1972
1973 Op->setReg(Dst);
1974 if (!TII->isOperandLegal(*UseMI, OpIdx, Op)) {
1975 Op->setReg(Reg);
1976 RS->eraseFromParent();
1977 return false;
1978 }
1979
1980 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
1981
1982 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
1983 // in which case we can erase them all later in runOnMachineFunction.
1984 if (MRI->use_nodbg_empty(MI.getOperand(0).getReg()))
1985 MI.eraseFromParent();
1986 return true;
1987 }
1988
1989 /// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
1990 /// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
1991 static bool isAGPRCopy(const SIRegisterInfo &TRI,
1992 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
1993 Register &OutReg, unsigned &OutSubReg) {
1994 assert(Copy.isCopy());
1995
1996 const MachineOperand &CopySrc = Copy.getOperand(1);
1997 Register CopySrcReg = CopySrc.getReg();
1998 if (!CopySrcReg.isVirtual())
1999 return false;
2000
2001 // Common case: copy from AGPR directly, e.g.
2002 // %1:vgpr_32 = COPY %0:agpr_32
2003 if (TRI.isAGPR(MRI, CopySrcReg)) {
2004 OutReg = CopySrcReg;
2005 OutSubReg = CopySrc.getSubReg();
2006 return true;
2007 }
2008
2009 // Sometimes it can also involve two copies, e.g.
2010 // %1:vgpr_256 = COPY %0:agpr_256
2011 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2012 const MachineInstr *CopySrcDef = MRI.getVRegDef(CopySrcReg);
2013 if (!CopySrcDef || !CopySrcDef->isCopy())
2014 return false;
2015
2016 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(1);
2017 Register OtherCopySrcReg = OtherCopySrc.getReg();
2018 if (!OtherCopySrcReg.isVirtual() ||
2019 CopySrcDef->getOperand(0).getSubReg() != AMDGPU::NoSubRegister ||
2020 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2021 !TRI.isAGPR(MRI, OtherCopySrcReg))
2022 return false;
2023
2024 OutReg = OtherCopySrcReg;
2025 OutSubReg = CopySrc.getSubReg();
2026 return true;
2027 }
2028
2029 // Try to hoist an AGPR to VGPR copy across a PHI.
2030 // This should allow folding of an AGPR into a consumer which may support it.
2031 //
2032 // Example 1: LCSSA PHI
2033 // loop:
2034 // %1:vreg = COPY %0:areg
2035 // exit:
2036 // %2:vreg = PHI %1:vreg, %loop
2037 // =>
2038 // loop:
2039 // exit:
2040 // %1:areg = PHI %0:areg, %loop
2041 // %2:vreg = COPY %1:areg
2042 //
2043 // Example 2: PHI with multiple incoming values:
2044 // entry:
2045 // %1:vreg = GLOBAL_LOAD(..)
2046 // loop:
2047 // %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
2048 // %3:areg = COPY %2:vreg
2049 // %4:areg = (instr using %3:areg)
2050 // %5:vreg = COPY %4:areg
2051 // =>
2052 // entry:
2053 // %1:vreg = GLOBAL_LOAD(..)
2054 // %2:areg = COPY %1:vreg
2055 // loop:
2056 // %3:areg = PHI %2:areg, %entry, %X:areg,
2057 // %4:areg = (instr using %3:areg)
2058 bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
2059 assert(PHI.isPHI());
2060
2061 Register PhiOut = PHI.getOperand(0).getReg();
2062 if (!TRI->isVGPR(*MRI, PhiOut))
2063 return false;
2064
2065 // Iterate once over all incoming values of the PHI to check if this PHI is
2066 // eligible, and determine the exact AGPR RC we'll target.
2067 const TargetRegisterClass *ARC = nullptr;
2068 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2069 MachineOperand &MO = PHI.getOperand(K);
2070 MachineInstr *Copy = MRI->getVRegDef(MO.getReg());
2071 if (!Copy || !Copy->isCopy())
2072 continue;
2073
2074 Register AGPRSrc;
2075 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
2076 if (!isAGPRCopy(*TRI, *MRI, *Copy, AGPRSrc, AGPRRegMask))
2077 continue;
2078
2079 const TargetRegisterClass *CopyInRC = MRI->getRegClass(AGPRSrc);
2080 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
2081 CopyInRC = SubRC;
2082
2083 if (ARC && !ARC->hasSubClassEq(CopyInRC))
2084 return false;
2085 ARC = CopyInRC;
2086 }
2087
2088 if (!ARC)
2089 return false;
2090
2091 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
2092
2093 // Rewrite the PHI's incoming values to ARC.
2094 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
2095 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2096 MachineOperand &MO = PHI.getOperand(K);
2097 Register Reg = MO.getReg();
2098
2099 MachineBasicBlock::iterator InsertPt;
2100 MachineBasicBlock *InsertMBB = nullptr;
2101
2102 // Look at the def of Reg, ignoring all copies.
2103 unsigned CopyOpc = AMDGPU::COPY;
2104 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
2105
2106 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
2107 // the copy was single-use, it will be removed by DCE later.
2108 if (Def->isCopy()) {
2109 Register AGPRSrc;
2110 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
2111 if (isAGPRCopy(*TRI, *MRI, *Def, AGPRSrc, AGPRSubReg)) {
2112 MO.setReg(AGPRSrc);
2113 MO.setSubReg(AGPRSubReg);
2114 continue;
2115 }
2116
2117 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
2118 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
2119 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
2120 // is unlikely to be profitable.
2121 //
2122 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
2123 MachineOperand &CopyIn = Def->getOperand(1);
2124 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(Reg) &&
2125 TRI->isSGPRReg(*MRI, CopyIn.getReg()))
2126 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2127 }
2128
2129 InsertMBB = Def->getParent();
2130 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(++Def->getIterator());
2131 } else {
2132 InsertMBB = PHI.getOperand(MO.getOperandNo() + 1).getMBB();
2133 InsertPt = InsertMBB->getFirstTerminator();
2134 }
2135
2136 Register NewReg = MRI->createVirtualRegister(ARC);
2137 MachineInstr *MI = BuildMI(*InsertMBB, InsertPt, PHI.getDebugLoc(),
2138 TII->get(CopyOpc), NewReg)
2139 .addReg(Reg);
2140 MO.setReg(NewReg);
2141
2142 (void)MI;
2143 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
2144 }
2145
2146 // Replace the PHI's result with a new register.
2147 Register NewReg = MRI->createVirtualRegister(ARC);
2148 PHI.getOperand(0).setReg(NewReg);
2149
2150 // COPY that new register back to the original PhiOut register. This COPY will
2151 // usually be folded out later.
2152 MachineBasicBlock *MBB = PHI.getParent();
2153 BuildMI(*MBB, MBB->getFirstNonPHI(), PHI.getDebugLoc(),
2154 TII->get(AMDGPU::COPY), PhiOut)
2155 .addReg(NewReg);
2156
2157 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
2158 return true;
2159 }
2160
2161 // Attempt to convert VGPR load to an AGPR load.
2162 bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
2163 assert(MI.mayLoad());
2164 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
2165 return false;
2166
2167 MachineOperand &Def = MI.getOperand(0);
2168 if (!Def.isDef())
2169 return false;
2170
2171 Register DefReg = Def.getReg();
2172
2173 if (DefReg.isPhysical() || !TRI->isVGPR(*MRI, DefReg))
2174 return false;
2175
2176 SmallVector<const MachineInstr*, 8> Users;
2177 SmallVector<Register, 8> MoveRegs;
2178 for (const MachineInstr &I : MRI->use_nodbg_instructions(DefReg))
2179 Users.push_back(&I);
2180
2181 if (Users.empty())
2182 return false;
2183
2184 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
2185 while (!Users.empty()) {
2186 const MachineInstr *I = Users.pop_back_val();
2187 if (!I->isCopy() && !I->isRegSequence())
2188 return false;
2189 Register DstReg = I->getOperand(0).getReg();
2190 // Physical registers may have more than one instruction definitions
2191 if (DstReg.isPhysical())
2192 return false;
2193 if (TRI->isAGPR(*MRI, DstReg))
2194 continue;
2195 MoveRegs.push_back(DstReg);
2196 for (const MachineInstr &U : MRI->use_nodbg_instructions(DstReg))
2197 Users.push_back(&U);
2198 }
2199
2200 const TargetRegisterClass *RC = MRI->getRegClass(DefReg);
2201 MRI->setRegClass(DefReg, TRI->getEquivalentAGPRClass(RC));
2202 if (!TII->isOperandLegal(MI, 0, &Def)) {
2203 MRI->setRegClass(DefReg, RC);
2204 return false;
2205 }
2206
2207 while (!MoveRegs.empty()) {
2208 Register Reg = MoveRegs.pop_back_val();
2209 MRI->setRegClass(Reg, TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg)));
2210 }
2211
2212 LLVM_DEBUG(dbgs() << "Folded " << MI);
2213
2214 return true;
2215 }
2216
2217 // tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
2218 // For GFX90A and later, this is pretty much always a good thing, but for GFX908
2219 // there's cases where it can create a lot more AGPR-AGPR copies, which are
2220 // expensive on this architecture due to the lack of V_ACCVGPR_MOV.
2221 //
2222 // This function looks at all AGPR PHIs in a basic block and collects their
2223 // operands. Then, it checks for register that are used more than once across
2224 // all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
2225 // having to create one VGPR temporary per use, which can get very messy if
2226 // these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
2227 // element).
2228 //
2229 // Example
2230 // a:
2231 // %in:agpr_256 = COPY %foo:vgpr_256
2232 // c:
2233 // %x:agpr_32 = ..
2234 // b:
2235 // %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
2236 // %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
2237 // %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
2238 // =>
2239 // a:
2240 // %in:agpr_256 = COPY %foo:vgpr_256
2241 // %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
2242 // %tmp_agpr:agpr_32 = COPY %tmp
2243 // c:
2244 // %x:agpr_32 = ..
2245 // b:
2246 // %0:areg = PHI %tmp_agpr, %a, %x, %c
2247 // %1:areg = PHI %tmp_agpr, %a, %y, %c
2248 // %2:areg = PHI %tmp_agpr, %a, %z, %c
2249 bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
2250 // This is only really needed on GFX908 where AGPR-AGPR copies are
2251 // unreasonably difficult.
2252 if (ST->hasGFX90AInsts())
2253 return false;
2254
2255 // Look at all AGPR Phis and collect the register + subregister used.
2256 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
2257 RegToMO;
2258
2259 for (auto &MI : MBB) {
2260 if (!MI.isPHI())
2261 break;
2262
2263 if (!TRI->isAGPR(*MRI, MI.getOperand(0).getReg()))
2264 continue;
2265
2266 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
2267 MachineOperand &PhiMO = MI.getOperand(K);
2268 if (!PhiMO.getSubReg())
2269 continue;
2270 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(&PhiMO);
2271 }
2272 }
2273
2274 // For all (Reg, SubReg) pair that are used more than once, cache the value in
2275 // a VGPR.
2276 bool Changed = false;
2277 for (const auto &[Entry, MOs] : RegToMO) {
2278 if (MOs.size() == 1)
2279 continue;
2280
2281 const auto [Reg, SubReg] = Entry;
2282 MachineInstr *Def = MRI->getVRegDef(Reg);
2283 MachineBasicBlock *DefMBB = Def->getParent();
2284
2285 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
2286 // out.
2287 const TargetRegisterClass *ARC = getRegOpRC(*MRI, *TRI, *MOs.front());
2288 Register TempVGPR =
2289 MRI->createVirtualRegister(TRI->getEquivalentVGPRClass(ARC));
2290 MachineInstr *VGPRCopy =
2291 BuildMI(*DefMBB, ++Def->getIterator(), Def->getDebugLoc(),
2292 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), TempVGPR)
2293 .addReg(Reg, /* flags */ 0, SubReg);
2294
2295 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
2296 Register TempAGPR = MRI->createVirtualRegister(ARC);
2297 BuildMI(*DefMBB, ++VGPRCopy->getIterator(), Def->getDebugLoc(),
2298 TII->get(AMDGPU::COPY), TempAGPR)
2299 .addReg(TempVGPR);
2300
2301 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
2302 for (MachineOperand *MO : MOs) {
2303 MO->setReg(TempAGPR);
2304 MO->setSubReg(AMDGPU::NoSubRegister);
2305 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
2306 }
2307
2308 Changed = true;
2309 }
2310
2311 return Changed;
2312 }
2313
2314 bool SIFoldOperandsImpl::run(MachineFunction &MF) {
2315 MRI = &MF.getRegInfo();
2316 ST = &MF.getSubtarget<GCNSubtarget>();
2317 TII = ST->getInstrInfo();
2318 TRI = &TII->getRegisterInfo();
2319 MFI = MF.getInfo<SIMachineFunctionInfo>();
2320
2321 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
2322 // correctly handle signed zeros.
2323 //
2324 // FIXME: Also need to check strictfp
2325 bool IsIEEEMode = MFI->getMode().IEEE;
2326 bool HasNSZ = MFI->hasNoSignedZerosFPMath();
2327
2328 bool Changed = false;
2329 for (MachineBasicBlock *MBB : depth_first(&MF)) {
2330 MachineOperand *CurrentKnownM0Val = nullptr;
2331 for (auto &MI : make_early_inc_range(*MBB)) {
2332 Changed |= tryFoldCndMask(MI);
2333
2334 if (tryFoldZeroHighBits(MI)) {
2335 Changed = true;
2336 continue;
2337 }
2338
2339 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
2340 Changed = true;
2341 continue;
2342 }
2343
2344 if (MI.isPHI() && tryFoldPhiAGPR(MI)) {
2345 Changed = true;
2346 continue;
2347 }
2348
2349 if (MI.mayLoad() && tryFoldLoad(MI)) {
2350 Changed = true;
2351 continue;
2352 }
2353
2354 if (TII->isFoldableCopy(MI)) {
2355 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
2356 continue;
2357 }
2358
2359 // Saw an unknown clobber of m0, so we no longer know what it is.
2360 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI))
2361 CurrentKnownM0Val = nullptr;
2362
2363 // TODO: Omod might be OK if there is NSZ only on the source
2364 // instruction, and not the omod multiply.
2365 if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) ||
2366 !tryFoldOMod(MI))
2367 Changed |= tryFoldClamp(MI);
2368 }
2369
2370 Changed |= tryOptimizeAGPRPhis(*MBB);
2371 }
2372
2373 return Changed;
2374 }
2375
2376 PreservedAnalyses SIFoldOperandsPass::run(MachineFunction &MF,
2377 MachineFunctionAnalysisManager &) {
2378 bool Changed = SIFoldOperandsImpl().run(MF);
2379 if (!Changed) {
2380 return PreservedAnalyses::all();
2381 }
2382 auto PA = getMachineFunctionPassPreservedAnalyses();
2383 PA.preserveSet<CFGAnalyses>();
2384 return PA;
2385 }
LLVM monorepo