lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp

   1 //===- SIPreAllocateWWMRegs.cpp - WWM Register Pre-allocation -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// Pass to pre-allocated WWM registers
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AMDGPU.h"
  15 #include "AMDGPUSubtarget.h"
  16 #include "SIInstrInfo.h"
  17 #include "SIRegisterInfo.h"
  18 #include "SIMachineFunctionInfo.h"
  19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  20 #include "llvm/ADT/PostOrderIterator.h"
  21 #include "llvm/CodeGen/VirtRegMap.h"
  22 #include "llvm/CodeGen/LiveInterval.h"
  23 #include "llvm/CodeGen/LiveIntervals.h"
  24 #include "llvm/CodeGen/LiveRegMatrix.h"
  25 #include "llvm/CodeGen/MachineDominators.h"
  26 #include "llvm/CodeGen/MachineFunctionPass.h"
  27 #include "llvm/CodeGen/RegisterClassInfo.h"
  28
  29 using namespace llvm;
  30
  31 #define DEBUG_TYPE "si-pre-allocate-wwm-regs"
  32
  33 namespace {
  34
  35 class SIPreAllocateWWMRegs : public MachineFunctionPass {
  36 private:
  37   const SIInstrInfo *TII;
  38   const SIRegisterInfo *TRI;
  39   MachineRegisterInfo *MRI;
  40   LiveIntervals *LIS;
  41   LiveRegMatrix *Matrix;
  42   VirtRegMap *VRM;
  43   RegisterClassInfo RegClassInfo;
  44
  45   std::vector<unsigned> RegsToRewrite;
  46
  47 public:
  48   static char ID;
  49
  50   SIPreAllocateWWMRegs() : MachineFunctionPass(ID) {
  51     initializeSIPreAllocateWWMRegsPass(*PassRegistry::getPassRegistry());
  52   }
  53
  54   bool runOnMachineFunction(MachineFunction &MF) override;
  55
  56   void getAnalysisUsage(AnalysisUsage &AU) const override {
  57     AU.addRequired<LiveIntervals>();
  58     AU.addPreserved<LiveIntervals>();
  59     AU.addRequired<VirtRegMap>();
  60     AU.addRequired<LiveRegMatrix>();
  61     AU.addPreserved<SlotIndexes>();
  62     AU.setPreservesCFG();
  63     MachineFunctionPass::getAnalysisUsage(AU);
  64   }
  65
  66 private:
  67   bool processDef(MachineOperand &MO);
  68   void rewriteRegs(MachineFunction &MF);
  69 };
  70
  71 } // End anonymous namespace.
  72
  73 INITIALIZE_PASS_BEGIN(SIPreAllocateWWMRegs, DEBUG_TYPE,
  74                 "SI Pre-allocate WWM Registers", false, false)
  75 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
  76 INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
  77 INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix)
  78 INITIALIZE_PASS_END(SIPreAllocateWWMRegs, DEBUG_TYPE,
  79                 "SI Pre-allocate WWM Registers", false, false)
  80
  81 char SIPreAllocateWWMRegs::ID = 0;
  82
  83 char &llvm::SIPreAllocateWWMRegsID = SIPreAllocateWWMRegs::ID;
  84
  85 FunctionPass *llvm::createSIPreAllocateWWMRegsPass() {
  86   return new SIPreAllocateWWMRegs();
  87 }
  88
  89 bool SIPreAllocateWWMRegs::processDef(MachineOperand &MO) {
  90   if (!MO.isReg())
  91     return false;
  92
  93   Register Reg = MO.getReg();
  94
  95   if (!TRI->isVGPR(*MRI, Reg))
  96     return false;
  97
  98   if (Register::isPhysicalRegister(Reg))
  99     return false;
 100
 101   if (VRM->hasPhys(Reg))
 102     return false;
 103
 104   LiveInterval &LI = LIS->getInterval(Reg);
 105
 106   for (unsigned PhysReg : RegClassInfo.getOrder(MRI->getRegClass(Reg))) {
 107     if (!MRI->isPhysRegUsed(PhysReg) &&
 108         Matrix->checkInterference(LI, PhysReg) == LiveRegMatrix::IK_Free) {
 109       Matrix->assign(LI, PhysReg);
 110       assert(PhysReg != 0);
 111       RegsToRewrite.push_back(Reg);
 112       return true;
 113     }
 114   }
 115
 116   llvm_unreachable("physreg not found for WWM expression");
 117   return false;
 118 }
 119
 120 void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {
 121   for (MachineBasicBlock &MBB : MF) {
 122     for (MachineInstr &MI : MBB) {
 123       for (MachineOperand &MO : MI.operands()) {
 124         if (!MO.isReg())
 125           continue;
 126
 127         const Register VirtReg = MO.getReg();
 128         if (Register::isPhysicalRegister(VirtReg))
 129           continue;
 130
 131         if (!VRM->hasPhys(VirtReg))
 132           continue;
 133
 134         Register PhysReg = VRM->getPhys(VirtReg);
 135         const unsigned SubReg = MO.getSubReg();
 136         if (SubReg != 0) {
 137           PhysReg = TRI->getSubReg(PhysReg, SubReg);
 138           MO.setSubReg(0);
 139         }
 140
 141         MO.setReg(PhysReg);
 142         MO.setIsRenamable(false);
 143       }
 144     }
 145   }
 146
 147   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 148
 149   for (unsigned Reg : RegsToRewrite) {
 150     LIS->removeInterval(Reg);
 151
 152     const Register PhysReg = VRM->getPhys(Reg);
 153     assert(PhysReg != 0);
 154     MFI->ReserveWWMRegister(PhysReg);
 155   }
 156
 157   RegsToRewrite.clear();
 158
 159   // Update the set of reserved registers to include WWM ones.
 160   MRI->freezeReservedRegs(MF);
 161 }
 162
 163 bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) {
 164   LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n");
 165
 166   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 167
 168   TII = ST.getInstrInfo();
 169   TRI = &TII->getRegisterInfo();
 170   MRI = &MF.getRegInfo();
 171
 172   LIS = &getAnalysis<LiveIntervals>();
 173   Matrix = &getAnalysis<LiveRegMatrix>();
 174   VRM = &getAnalysis<VirtRegMap>();
 175
 176   RegClassInfo.runOnMachineFunction(MF);
 177
 178   bool RegsAssigned = false;
 179
 180   // We use a reverse post-order traversal of the control-flow graph to
 181   // guarantee that we visit definitions in dominance order. Since WWM
 182   // expressions are guaranteed to never involve phi nodes, and we can only
 183   // escape WWM through the special WWM instruction, this means that this is a
 184   // perfect elimination order, so we can never do any better.
 185   ReversePostOrderTraversal<MachineFunction*> RPOT(&MF);
 186
 187   for (MachineBasicBlock *MBB : RPOT) {
 188     bool InWWM = false;
 189     for (MachineInstr &MI : *MBB) {
 190       if (MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
 191           MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B64)
 192         RegsAssigned |= processDef(MI.getOperand(0));
 193
 194       if (MI.getOpcode() == AMDGPU::ENTER_WWM) {
 195         LLVM_DEBUG(dbgs() << "entering WWM region: " << MI << "\n");
 196         InWWM = true;
 197         continue;
 198       }
 199
 200       if (MI.getOpcode() == AMDGPU::EXIT_WWM) {
 201         LLVM_DEBUG(dbgs() << "exiting WWM region: " << MI << "\n");
 202         InWWM = false;
 203       }
 204
 205       if (!InWWM)
 206         continue;
 207
 208       LLVM_DEBUG(dbgs() << "processing " << MI << "\n");
 209
 210       for (MachineOperand &DefOpnd : MI.defs()) {
 211         RegsAssigned |= processDef(DefOpnd);
 212       }
 213     }
 214   }
 215
 216   if (!RegsAssigned)
 217     return false;
 218
 219   rewriteRegs(MF);
 220   return true;
 221 }