lib/Target/AMDGPU/SIFixSGPRCopies.cpp

   1 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// Copies from VGPR to SGPR registers are illegal and the register coalescer
  11 /// will sometimes generate these illegal copies in situations like this:
  12 ///
  13 ///  Register Class <vsrc> is the union of <vgpr> and <sgpr>
  14 ///
  15 /// BB0:
  16 ///   %0 <sgpr> = SCALAR_INST
  17 ///   %1 <vsrc> = COPY %0 <sgpr>
  18 ///    ...
  19 ///    BRANCH %cond BB1, BB2
  20 ///  BB1:
  21 ///    %2 <vgpr> = VECTOR_INST
  22 ///    %3 <vsrc> = COPY %2 <vgpr>
  23 ///  BB2:
  24 ///    %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
  25 ///    %5 <vgpr> = VECTOR_INST %4 <vsrc>
  26 ///
  27 ///
  28 /// The coalescer will begin at BB0 and eliminate its copy, then the resulting
  29 /// code will look like this:
  30 ///
  31 /// BB0:
  32 ///   %0 <sgpr> = SCALAR_INST
  33 ///    ...
  34 ///    BRANCH %cond BB1, BB2
  35 /// BB1:
  36 ///   %2 <vgpr> = VECTOR_INST
  37 ///   %3 <vsrc> = COPY %2 <vgpr>
  38 /// BB2:
  39 ///   %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
  40 ///   %5 <vgpr> = VECTOR_INST %4 <sgpr>
  41 ///
  42 /// Now that the result of the PHI instruction is an SGPR, the register
  43 /// allocator is now forced to constrain the register class of %3 to
  44 /// <sgpr> so we end up with final code like this:
  45 ///
  46 /// BB0:
  47 ///   %0 <sgpr> = SCALAR_INST
  48 ///    ...
  49 ///    BRANCH %cond BB1, BB2
  50 /// BB1:
  51 ///   %2 <vgpr> = VECTOR_INST
  52 ///   %3 <sgpr> = COPY %2 <vgpr>
  53 /// BB2:
  54 ///   %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
  55 ///   %5 <vgpr> = VECTOR_INST %4 <sgpr>
  56 ///
  57 /// Now this code contains an illegal copy from a VGPR to an SGPR.
  58 ///
  59 /// In order to avoid this problem, this pass searches for PHI instructions
  60 /// which define a <vsrc> register and constrains its definition class to
  61 /// <vgpr> if the user of the PHI's definition register is a vector instruction.
  62 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
  63 /// will be unable to perform the COPY removal from the above example  which
  64 /// ultimately led to the creation of an illegal COPY.
  65 //===----------------------------------------------------------------------===//
  66
  67 #include "AMDGPU.h"
  68 #include "AMDGPUSubtarget.h"
  69 #include "SIInstrInfo.h"
  70 #include "SIRegisterInfo.h"
  71 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  72 #include "llvm/ADT/DenseSet.h"
  73 #include "llvm/ADT/STLExtras.h"
  74 #include "llvm/ADT/SmallSet.h"
  75 #include "llvm/ADT/SmallVector.h"
  76 #include "llvm/CodeGen/MachineBasicBlock.h"
  77 #include "llvm/CodeGen/MachineDominators.h"
  78 #include "llvm/CodeGen/MachineFunction.h"
  79 #include "llvm/CodeGen/MachineFunctionPass.h"
  80 #include "llvm/CodeGen/MachineInstr.h"
  81 #include "llvm/CodeGen/MachineInstrBuilder.h"
  82 #include "llvm/CodeGen/MachineOperand.h"
  83 #include "llvm/CodeGen/MachineRegisterInfo.h"
  84 #include "llvm/CodeGen/TargetRegisterInfo.h"
  85 #include "llvm/Pass.h"
  86 #include "llvm/Support/CodeGen.h"
  87 #include "llvm/Support/CommandLine.h"
  88 #include "llvm/Support/Debug.h"
  89 #include "llvm/Support/raw_ostream.h"
  90 #include "llvm/Target/TargetMachine.h"
  91 #include <cassert>
  92 #include <cstdint>
  93 #include <iterator>
  94 #include <list>
  95 #include <map>
  96 #include <tuple>
  97 #include <utility>
  98
  99 using namespace llvm;
 100
 101 #define DEBUG_TYPE "si-fix-sgpr-copies"
 102
 103 static cl::opt<bool> EnableM0Merge(
 104   "amdgpu-enable-merge-m0",
 105   cl::desc("Merge and hoist M0 initializations"),
 106   cl::init(true));
 107
 108 namespace {
 109
 110 class SIFixSGPRCopies : public MachineFunctionPass {
 111   MachineDominatorTree *MDT;
 112
 113 public:
 114   static char ID;
 115
 116   SIFixSGPRCopies() : MachineFunctionPass(ID) {}
 117
 118   bool runOnMachineFunction(MachineFunction &MF) override;
 119
 120   StringRef getPassName() const override { return "SI Fix SGPR copies"; }
 121
 122   void getAnalysisUsage(AnalysisUsage &AU) const override {
 123     AU.addRequired<MachineDominatorTree>();
 124     AU.addPreserved<MachineDominatorTree>();
 125     AU.setPreservesCFG();
 126     MachineFunctionPass::getAnalysisUsage(AU);
 127   }
 128 };
 129
 130 } // end anonymous namespace
 131
 132 INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
 133                      "SI Fix SGPR copies", false, false)
 134 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
 135 INITIALIZE_PASS_END(SIFixSGPRCopies, DEBUG_TYPE,
 136                      "SI Fix SGPR copies", false, false)
 137
 138 char SIFixSGPRCopies::ID = 0;
 139
 140 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
 141
 142 FunctionPass *llvm::createSIFixSGPRCopiesPass() {
 143   return new SIFixSGPRCopies();
 144 }
 145
 146 static bool hasVectorOperands(const MachineInstr &MI,
 147                               const SIRegisterInfo *TRI) {
 148   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
 149   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
 150     if (!MI.getOperand(i).isReg() ||
 151         !Register::isVirtualRegister(MI.getOperand(i).getReg()))
 152       continue;
 153
 154     if (TRI->hasVectorRegisters(MRI.getRegClass(MI.getOperand(i).getReg())))
 155       return true;
 156   }
 157   return false;
 158 }
 159
 160 static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
 161 getCopyRegClasses(const MachineInstr &Copy,
 162                   const SIRegisterInfo &TRI,
 163                   const MachineRegisterInfo &MRI) {
 164   Register DstReg = Copy.getOperand(0).getReg();
 165   Register SrcReg = Copy.getOperand(1).getReg();
 166
 167   const TargetRegisterClass *SrcRC = Register::isVirtualRegister(SrcReg)
 168                                          ? MRI.getRegClass(SrcReg)
 169                                          : TRI.getPhysRegClass(SrcReg);
 170
 171   // We don't really care about the subregister here.
 172   // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
 173
 174   const TargetRegisterClass *DstRC = Register::isVirtualRegister(DstReg)
 175                                          ? MRI.getRegClass(DstReg)
 176                                          : TRI.getPhysRegClass(DstReg);
 177
 178   return std::make_pair(SrcRC, DstRC);
 179 }
 180
 181 static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
 182                              const TargetRegisterClass *DstRC,
 183                              const SIRegisterInfo &TRI) {
 184   return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
 185          TRI.hasVectorRegisters(SrcRC);
 186 }
 187
 188 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
 189                              const TargetRegisterClass *DstRC,
 190                              const SIRegisterInfo &TRI) {
 191   return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
 192          TRI.hasVectorRegisters(DstRC);
 193 }
 194
 195 static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
 196                                       const SIRegisterInfo *TRI,
 197                                       const SIInstrInfo *TII) {
 198   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
 199   auto &Src = MI.getOperand(1);
 200   Register DstReg = MI.getOperand(0).getReg();
 201   Register SrcReg = Src.getReg();
 202   if (!Register::isVirtualRegister(SrcReg) ||
 203       !Register::isVirtualRegister(DstReg))
 204     return false;
 205
 206   for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
 207     const auto *UseMI = MO.getParent();
 208     if (UseMI == &MI)
 209       continue;
 210     if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
 211         UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
 212         !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src))
 213       return false;
 214   }
 215   // Change VGPR to SGPR destination.
 216   MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
 217   return true;
 218 }
 219
 220 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
 221 //
 222 // SGPRx = ...
 223 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
 224 // VGPRz = COPY SGPRy
 225 //
 226 // ==>
 227 //
 228 // VGPRx = COPY SGPRx
 229 // VGPRz = REG_SEQUENCE VGPRx, sub0
 230 //
 231 // This exposes immediate folding opportunities when materializing 64-bit
 232 // immediates.
 233 static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
 234                                         const SIRegisterInfo *TRI,
 235                                         const SIInstrInfo *TII,
 236                                         MachineRegisterInfo &MRI) {
 237   assert(MI.isRegSequence());
 238
 239   Register DstReg = MI.getOperand(0).getReg();
 240   if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
 241     return false;
 242
 243   if (!MRI.hasOneUse(DstReg))
 244     return false;
 245
 246   MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
 247   if (!CopyUse.isCopy())
 248     return false;
 249
 250   // It is illegal to have vreg inputs to a physreg defining reg_sequence.
 251   if (Register::isPhysicalRegister(CopyUse.getOperand(0).getReg()))
 252     return false;
 253
 254   const TargetRegisterClass *SrcRC, *DstRC;
 255   std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
 256
 257   if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
 258     return false;
 259
 260   if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
 261     return true;
 262
 263   // TODO: Could have multiple extracts?
 264   unsigned SubReg = CopyUse.getOperand(1).getSubReg();
 265   if (SubReg != AMDGPU::NoSubRegister)
 266     return false;
 267
 268   MRI.setRegClass(DstReg, DstRC);
 269
 270   // SGPRx = ...
 271   // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
 272   // VGPRz = COPY SGPRy
 273
 274   // =>
 275   // VGPRx = COPY SGPRx
 276   // VGPRz = REG_SEQUENCE VGPRx, sub0
 277
 278   MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
 279   bool IsAGPR = TRI->hasAGPRs(DstRC);
 280
 281   for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
 282     Register SrcReg = MI.getOperand(I).getReg();
 283     unsigned SrcSubReg = MI.getOperand(I).getSubReg();
 284
 285     const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
 286     assert(TRI->isSGPRClass(SrcRC) &&
 287            "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
 288
 289     SrcRC = TRI->getSubRegClass(SrcRC, SrcSubReg);
 290     const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
 291
 292     Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
 293
 294     BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
 295             TmpReg)
 296         .add(MI.getOperand(I));
 297
 298     if (IsAGPR) {
 299       const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
 300       Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
 301       unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
 302         AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
 303       BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
 304             TmpAReg)
 305         .addReg(TmpReg, RegState::Kill);
 306       TmpReg = TmpAReg;
 307     }
 308
 309     MI.getOperand(I).setReg(TmpReg);
 310   }
 311
 312   CopyUse.eraseFromParent();
 313   return true;
 314 }
 315
 316 static bool phiHasVGPROperands(const MachineInstr &PHI,
 317                                const MachineRegisterInfo &MRI,
 318                                const SIRegisterInfo *TRI,
 319                                const SIInstrInfo *TII) {
 320   for (unsigned i = 1; i < PHI.getNumOperands(); i += 2) {
 321     Register Reg = PHI.getOperand(i).getReg();
 322     if (TRI->hasVGPRs(MRI.getRegClass(Reg)))
 323       return true;
 324   }
 325   return false;
 326 }
 327
 328 static bool phiHasBreakDef(const MachineInstr &PHI,
 329                            const MachineRegisterInfo &MRI,
 330                            SmallSet<unsigned, 8> &Visited) {
 331   for (unsigned i = 1; i < PHI.getNumOperands(); i += 2) {
 332     Register Reg = PHI.getOperand(i).getReg();
 333     if (Visited.count(Reg))
 334       continue;
 335
 336     Visited.insert(Reg);
 337
 338     MachineInstr *DefInstr = MRI.getVRegDef(Reg);
 339     switch (DefInstr->getOpcode()) {
 340     default:
 341       break;
 342     case AMDGPU::SI_IF_BREAK:
 343       return true;
 344     case AMDGPU::PHI:
 345       if (phiHasBreakDef(*DefInstr, MRI, Visited))
 346         return true;
 347     }
 348   }
 349   return false;
 350 }
 351
 352 static bool hasTerminatorThatModifiesExec(const MachineBasicBlock &MBB,
 353                                           const TargetRegisterInfo &TRI) {
 354   for (MachineBasicBlock::const_iterator I = MBB.getFirstTerminator(),
 355        E = MBB.end(); I != E; ++I) {
 356     if (I->modifiesRegister(AMDGPU::EXEC, &TRI))
 357       return true;
 358   }
 359   return false;
 360 }
 361
 362 static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
 363                                     const MachineInstr *MoveImm,
 364                                     const SIInstrInfo *TII,
 365                                     unsigned &SMovOp,
 366                                     int64_t &Imm) {
 367   if (Copy->getOpcode() != AMDGPU::COPY)
 368     return false;
 369
 370   if (!MoveImm->isMoveImmediate())
 371     return false;
 372
 373   const MachineOperand *ImmOp =
 374       TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
 375   if (!ImmOp->isImm())
 376     return false;
 377
 378   // FIXME: Handle copies with sub-regs.
 379   if (Copy->getOperand(0).getSubReg())
 380     return false;
 381
 382   switch (MoveImm->getOpcode()) {
 383   default:
 384     return false;
 385   case AMDGPU::V_MOV_B32_e32:
 386     SMovOp = AMDGPU::S_MOV_B32;
 387     break;
 388   case AMDGPU::V_MOV_B64_PSEUDO:
 389     SMovOp = AMDGPU::S_MOV_B64;
 390     break;
 391   }
 392   Imm = ImmOp->getImm();
 393   return true;
 394 }
 395
 396 template <class UnaryPredicate>
 397 bool searchPredecessors(const MachineBasicBlock *MBB,
 398                         const MachineBasicBlock *CutOff,
 399                         UnaryPredicate Predicate) {
 400   if (MBB == CutOff)
 401     return false;
 402
 403   DenseSet<const MachineBasicBlock *> Visited;
 404   SmallVector<MachineBasicBlock *, 4> Worklist(MBB->pred_begin(),
 405                                                MBB->pred_end());
 406
 407   while (!Worklist.empty()) {
 408     MachineBasicBlock *MBB = Worklist.pop_back_val();
 409
 410     if (!Visited.insert(MBB).second)
 411       continue;
 412     if (MBB == CutOff)
 413       continue;
 414     if (Predicate(MBB))
 415       return true;
 416
 417     Worklist.append(MBB->pred_begin(), MBB->pred_end());
 418   }
 419
 420   return false;
 421 }
 422
 423 static bool predsHasDivergentTerminator(MachineBasicBlock *MBB,
 424                                         const TargetRegisterInfo *TRI) {
 425   return searchPredecessors(MBB, nullptr, [TRI](MachineBasicBlock *MBB) {
 426            return hasTerminatorThatModifiesExec(*MBB, *TRI); });
 427 }
 428
 429 // Checks if there is potential path From instruction To instruction.
 430 // If CutOff is specified and it sits in between of that path we ignore
 431 // a higher portion of the path and report it is not reachable.
 432 static bool isReachable(const MachineInstr *From,
 433                         const MachineInstr *To,
 434                         const MachineBasicBlock *CutOff,
 435                         MachineDominatorTree &MDT) {
 436   // If either From block dominates To block or instructions are in the same
 437   // block and From is higher.
 438   if (MDT.dominates(From, To))
 439     return true;
 440
 441   const MachineBasicBlock *MBBFrom = From->getParent();
 442   const MachineBasicBlock *MBBTo = To->getParent();
 443   if (MBBFrom == MBBTo)
 444     return false;
 445
 446   // Instructions are in different blocks, do predecessor search.
 447   // We should almost never get here since we do not usually produce M0 stores
 448   // other than -1.
 449   return searchPredecessors(MBBTo, CutOff, [MBBFrom]
 450            (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
 451 }
 452
 453 // Return the first non-prologue instruction in the block.
 454 static MachineBasicBlock::iterator
 455 getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
 456   MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
 457   while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
 458     ++I;
 459
 460   return I;
 461 }
 462
 463 // Hoist and merge identical SGPR initializations into a common predecessor.
 464 // This is intended to combine M0 initializations, but can work with any
 465 // SGPR. A VGPR cannot be processed since we cannot guarantee vector
 466 // executioon.
 467 static bool hoistAndMergeSGPRInits(unsigned Reg,
 468                                    const MachineRegisterInfo &MRI,
 469                                    const TargetRegisterInfo *TRI,
 470                                    MachineDominatorTree &MDT,
 471                                    const TargetInstrInfo *TII) {
 472   // List of inits by immediate value.
 473   using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
 474   InitListMap Inits;
 475   // List of clobbering instructions.
 476   SmallVector<MachineInstr*, 8> Clobbers;
 477   // List of instructions marked for deletion.
 478   SmallSet<MachineInstr*, 8> MergedInstrs;
 479
 480   bool Changed = false;
 481
 482   for (auto &MI : MRI.def_instructions(Reg)) {
 483     MachineOperand *Imm = nullptr;
 484     for (auto &MO : MI.operands()) {
 485       if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
 486           (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
 487         Imm = nullptr;
 488         break;
 489       } else if (MO.isImm())
 490         Imm = &MO;
 491     }
 492     if (Imm)
 493       Inits[Imm->getImm()].push_front(&MI);
 494     else
 495       Clobbers.push_back(&MI);
 496   }
 497
 498   for (auto &Init : Inits) {
 499     auto &Defs = Init.second;
 500
 501     for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
 502       MachineInstr *MI1 = *I1;
 503
 504       for (auto I2 = std::next(I1); I2 != E; ) {
 505         MachineInstr *MI2 = *I2;
 506
 507         // Check any possible interference
 508         auto interferes = [&](MachineBasicBlock::iterator From,
 509                               MachineBasicBlock::iterator To) -> bool {
 510
 511           assert(MDT.dominates(&*To, &*From));
 512
 513           auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
 514             const MachineBasicBlock *MBBFrom = From->getParent();
 515             const MachineBasicBlock *MBBTo = To->getParent();
 516             bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
 517             bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
 518             if (!MayClobberFrom && !MayClobberTo)
 519               return false;
 520             if ((MayClobberFrom && !MayClobberTo) ||
 521                 (!MayClobberFrom && MayClobberTo))
 522               return true;
 523             // Both can clobber, this is not an interference only if both are
 524             // dominated by Clobber and belong to the same block or if Clobber
 525             // properly dominates To, given that To >> From, so it dominates
 526             // both and located in a common dominator.
 527             return !((MBBFrom == MBBTo &&
 528                       MDT.dominates(Clobber, &*From) &&
 529                       MDT.dominates(Clobber, &*To)) ||
 530                      MDT.properlyDominates(Clobber->getParent(), MBBTo));
 531           };
 532
 533           return (llvm::any_of(Clobbers, interferes)) ||
 534                  (llvm::any_of(Inits, [&](InitListMap::value_type &C) {
 535                     return C.first != Init.first &&
 536                            llvm::any_of(C.second, interferes);
 537                   }));
 538         };
 539
 540         if (MDT.dominates(MI1, MI2)) {
 541           if (!interferes(MI2, MI1)) {
 542             LLVM_DEBUG(dbgs()
 543                        << "Erasing from "
 544                        << printMBBReference(*MI2->getParent()) << " " << *MI2);
 545             MergedInstrs.insert(MI2);
 546             Changed = true;
 547             ++I2;
 548             continue;
 549           }
 550         } else if (MDT.dominates(MI2, MI1)) {
 551           if (!interferes(MI1, MI2)) {
 552             LLVM_DEBUG(dbgs()
 553                        << "Erasing from "
 554                        << printMBBReference(*MI1->getParent()) << " " << *MI1);
 555             MergedInstrs.insert(MI1);
 556             Changed = true;
 557             ++I1;
 558             break;
 559           }
 560         } else {
 561           auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
 562                                                      MI2->getParent());
 563           if (!MBB) {
 564             ++I2;
 565             continue;
 566           }
 567
 568           MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
 569           if (!interferes(MI1, I) && !interferes(MI2, I)) {
 570             LLVM_DEBUG(dbgs()
 571                        << "Erasing from "
 572                        << printMBBReference(*MI1->getParent()) << " " << *MI1
 573                        << "and moving from "
 574                        << printMBBReference(*MI2->getParent()) << " to "
 575                        << printMBBReference(*I->getParent()) << " " << *MI2);
 576             I->getParent()->splice(I, MI2->getParent(), MI2);
 577             MergedInstrs.insert(MI1);
 578             Changed = true;
 579             ++I1;
 580             break;
 581           }
 582         }
 583         ++I2;
 584       }
 585       ++I1;
 586     }
 587   }
 588
 589   // Remove initializations that were merged into another.
 590   for (auto &Init : Inits) {
 591     auto &Defs = Init.second;
 592     auto I = Defs.begin();
 593     while (I != Defs.end()) {
 594       if (MergedInstrs.count(*I)) {
 595         (*I)->eraseFromParent();
 596         I = Defs.erase(I);
 597       } else
 598         ++I;
 599     }
 600   }
 601
 602   // Try to schedule SGPR initializations as early as possible in the MBB.
 603   for (auto &Init : Inits) {
 604     auto &Defs = Init.second;
 605     for (auto MI : Defs) {
 606       auto MBB = MI->getParent();
 607       MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
 608       MachineBasicBlock::reverse_iterator B(BoundaryMI);
 609       // Check if B should actually be a bondary. If not set the previous
 610       // instruction as the boundary instead.
 611       if (!TII->isBasicBlockPrologue(*B))
 612         B++;
 613
 614       auto R = std::next(MI->getReverseIterator());
 615       const unsigned Threshold = 50;
 616       // Search until B or Threashold for a place to insert the initialization.
 617       for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
 618         if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
 619             TII->isSchedulingBoundary(*R, MBB, *MBB->getParent()))
 620           break;
 621
 622       // Move to directly after R.
 623       if (&*--R != MI)
 624         MBB->splice(*R, MBB, MI);
 625     }
 626   }
 627
 628   if (Changed)
 629     MRI.clearKillFlags(Reg);
 630
 631   return Changed;
 632 }
 633
 634 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
 635   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 636   MachineRegisterInfo &MRI = MF.getRegInfo();
 637   const SIRegisterInfo *TRI = ST.getRegisterInfo();
 638   const SIInstrInfo *TII = ST.getInstrInfo();
 639   MDT = &getAnalysis<MachineDominatorTree>();
 640
 641   SmallVector<MachineInstr *, 16> Worklist;
 642
 643   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
 644                                                   BI != BE; ++BI) {
 645     MachineBasicBlock &MBB = *BI;
 646     for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
 647          I != E; ++I) {
 648       MachineInstr &MI = *I;
 649
 650       switch (MI.getOpcode()) {
 651       default:
 652         continue;
 653       case AMDGPU::COPY:
 654       case AMDGPU::WQM:
 655       case AMDGPU::SOFT_WQM:
 656       case AMDGPU::WWM: {
 657         Register DstReg = MI.getOperand(0).getReg();
 658
 659         const TargetRegisterClass *SrcRC, *DstRC;
 660         std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, MRI);
 661
 662         if (!Register::isVirtualRegister(DstReg)) {
 663           // If the destination register is a physical register there isn't
 664           // really much we can do to fix this.
 665           // Some special instructions use M0 as an input. Some even only use
 666           // the first lane. Insert a readfirstlane and hope for the best.
 667           if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(SrcRC)) {
 668             Register TmpReg
 669               = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 670
 671             BuildMI(MBB, MI, MI.getDebugLoc(),
 672                     TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
 673               .add(MI.getOperand(1));
 674             MI.getOperand(1).setReg(TmpReg);
 675           }
 676
 677           continue;
 678         }
 679
 680         if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) {
 681           Register SrcReg = MI.getOperand(1).getReg();
 682           if (!Register::isVirtualRegister(SrcReg)) {
 683             TII->moveToVALU(MI, MDT);
 684             break;
 685           }
 686
 687           MachineInstr *DefMI = MRI.getVRegDef(SrcReg);
 688           unsigned SMovOp;
 689           int64_t Imm;
 690           // If we are just copying an immediate, we can replace the copy with
 691           // s_mov_b32.
 692           if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
 693             MI.getOperand(1).ChangeToImmediate(Imm);
 694             MI.addImplicitDefUseOperands(MF);
 695             MI.setDesc(TII->get(SMovOp));
 696             break;
 697           }
 698           TII->moveToVALU(MI, MDT);
 699         } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
 700           tryChangeVGPRtoSGPRinCopy(MI, TRI, TII);
 701         }
 702
 703         break;
 704       }
 705       case AMDGPU::PHI: {
 706         Register Reg = MI.getOperand(0).getReg();
 707         if (!TRI->isSGPRClass(MRI.getRegClass(Reg)))
 708           break;
 709
 710         // We don't need to fix the PHI if the common dominator of the
 711         // two incoming blocks terminates with a uniform branch.
 712         bool HasVGPROperand = phiHasVGPROperands(MI, MRI, TRI, TII);
 713         if (MI.getNumExplicitOperands() == 5 && !HasVGPROperand) {
 714           MachineBasicBlock *MBB0 = MI.getOperand(2).getMBB();
 715           MachineBasicBlock *MBB1 = MI.getOperand(4).getMBB();
 716
 717           if (!predsHasDivergentTerminator(MBB0, TRI) &&
 718               !predsHasDivergentTerminator(MBB1, TRI)) {
 719             LLVM_DEBUG(dbgs()
 720                        << "Not fixing PHI for uniform branch: " << MI << '\n');
 721             break;
 722           }
 723         }
 724
 725         // If a PHI node defines an SGPR and any of its operands are VGPRs,
 726         // then we need to move it to the VALU.
 727         //
 728         // Also, if a PHI node defines an SGPR and has all SGPR operands
 729         // we must move it to the VALU, because the SGPR operands will
 730         // all end up being assigned the same register, which means
 731         // there is a potential for a conflict if different threads take
 732         // different control flow paths.
 733         //
 734         // For Example:
 735         //
 736         // sgpr0 = def;
 737         // ...
 738         // sgpr1 = def;
 739         // ...
 740         // sgpr2 = PHI sgpr0, sgpr1
 741         // use sgpr2;
 742         //
 743         // Will Become:
 744         //
 745         // sgpr2 = def;
 746         // ...
 747         // sgpr2 = def;
 748         // ...
 749         // use sgpr2
 750         //
 751         // The one exception to this rule is when one of the operands
 752         // is defined by a SI_BREAK, SI_IF_BREAK, or SI_ELSE_BREAK
 753         // instruction.  In this case, there we know the program will
 754         // never enter the second block (the loop) without entering
 755         // the first block (where the condition is computed), so there
 756         // is no chance for values to be over-written.
 757
 758         SmallSet<unsigned, 8> Visited;
 759         if (HasVGPROperand || !phiHasBreakDef(MI, MRI, Visited)) {
 760           LLVM_DEBUG(dbgs() << "Fixing PHI: " << MI);
 761           TII->moveToVALU(MI, MDT);
 762         }
 763
 764         break;
 765       }
 766       case AMDGPU::REG_SEQUENCE:
 767         if (TRI->hasVectorRegisters(TII->getOpRegClass(MI, 0)) ||
 768             !hasVectorOperands(MI, TRI)) {
 769           foldVGPRCopyIntoRegSequence(MI, TRI, TII, MRI);
 770           continue;
 771         }
 772
 773         LLVM_DEBUG(dbgs() << "Fixing REG_SEQUENCE: " << MI);
 774
 775         TII->moveToVALU(MI, MDT);
 776         break;
 777       case AMDGPU::INSERT_SUBREG: {
 778         const TargetRegisterClass *DstRC, *Src0RC, *Src1RC;
 779         DstRC = MRI.getRegClass(MI.getOperand(0).getReg());
 780         Src0RC = MRI.getRegClass(MI.getOperand(1).getReg());
 781         Src1RC = MRI.getRegClass(MI.getOperand(2).getReg());
 782         if (TRI->isSGPRClass(DstRC) &&
 783             (TRI->hasVectorRegisters(Src0RC) ||
 784              TRI->hasVectorRegisters(Src1RC))) {
 785           LLVM_DEBUG(dbgs() << " Fixing INSERT_SUBREG: " << MI);
 786           TII->moveToVALU(MI, MDT);
 787         }
 788         break;
 789       }
 790       }
 791     }
 792   }
 793
 794   if (MF.getTarget().getOptLevel() > CodeGenOpt::None && EnableM0Merge)
 795     hoistAndMergeSGPRInits(AMDGPU::M0, MRI, TRI, *MDT, TII);
 796
 797   return true;
 798 }