lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp

   1 //===-- SIOptimizeExecMaskingPreRA.cpp ------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This pass removes redundant S_OR_B64 instructions enabling lanes in
  11 /// the exec. If two SI_END_CF (lowered as S_OR_B64) come together without any
  12 /// vector instructions between them we can only keep outer SI_END_CF, given
  13 /// that CFG is structured and exec bits of the outer end statement are always
  14 /// not less than exec bit of the inner one.
  15 ///
  16 /// This needs to be done before the RA to eliminate saved exec bits registers
  17 /// but after register coalescer to have no vector registers copies in between
  18 /// of different end cf statements.
  19 ///
  20 //===----------------------------------------------------------------------===//
  21
  22 #include "AMDGPU.h"
  23 #include "AMDGPUSubtarget.h"
  24 #include "SIInstrInfo.h"
  25 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  26 #include "llvm/CodeGen/LiveIntervals.h"
  27 #include "llvm/CodeGen/MachineFunctionPass.h"
  28
  29 using namespace llvm;
  30
  31 #define DEBUG_TYPE "si-optimize-exec-masking-pre-ra"
  32
  33 namespace {
  34
  35 class SIOptimizeExecMaskingPreRA : public MachineFunctionPass {
  36 public:
  37   static char ID;
  38
  39 public:
  40   SIOptimizeExecMaskingPreRA() : MachineFunctionPass(ID) {
  41     initializeSIOptimizeExecMaskingPreRAPass(*PassRegistry::getPassRegistry());
  42   }
  43
  44   bool runOnMachineFunction(MachineFunction &MF) override;
  45
  46   StringRef getPassName() const override {
  47     return "SI optimize exec mask operations pre-RA";
  48   }
  49
  50   void getAnalysisUsage(AnalysisUsage &AU) const override {
  51     AU.addRequired<LiveIntervals>();
  52     AU.setPreservesAll();
  53     MachineFunctionPass::getAnalysisUsage(AU);
  54   }
  55 };
  56
  57 } // End anonymous namespace.
  58
  59 INITIALIZE_PASS_BEGIN(SIOptimizeExecMaskingPreRA, DEBUG_TYPE,
  60                       "SI optimize exec mask operations pre-RA", false, false)
  61 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
  62 INITIALIZE_PASS_END(SIOptimizeExecMaskingPreRA, DEBUG_TYPE,
  63                     "SI optimize exec mask operations pre-RA", false, false)
  64
  65 char SIOptimizeExecMaskingPreRA::ID = 0;
  66
  67 char &llvm::SIOptimizeExecMaskingPreRAID = SIOptimizeExecMaskingPreRA::ID;
  68
  69 FunctionPass *llvm::createSIOptimizeExecMaskingPreRAPass() {
  70   return new SIOptimizeExecMaskingPreRA();
  71 }
  72
  73 static bool isEndCF(const MachineInstr& MI, const SIRegisterInfo* TRI) {
  74   return MI.getOpcode() == AMDGPU::S_OR_B64 &&
  75          MI.modifiesRegister(AMDGPU::EXEC, TRI);
  76 }
  77
  78 static bool isFullExecCopy(const MachineInstr& MI) {
  79   return MI.isFullCopy() && MI.getOperand(1).getReg() == AMDGPU::EXEC;
  80 }
  81
  82 static unsigned getOrNonExecReg(const MachineInstr &MI,
  83                                 const SIInstrInfo &TII) {
  84   auto Op = TII.getNamedOperand(MI, AMDGPU::OpName::src1);
  85   if (Op->isReg() && Op->getReg() != AMDGPU::EXEC)
  86      return Op->getReg();
  87   Op = TII.getNamedOperand(MI, AMDGPU::OpName::src0);
  88   if (Op->isReg() && Op->getReg() != AMDGPU::EXEC)
  89      return Op->getReg();
  90   return AMDGPU::NoRegister;
  91 }
  92
  93 static MachineInstr* getOrExecSource(const MachineInstr &MI,
  94                                      const SIInstrInfo &TII,
  95                                      const MachineRegisterInfo &MRI) {
  96   auto SavedExec = getOrNonExecReg(MI, TII);
  97   if (SavedExec == AMDGPU::NoRegister)
  98     return nullptr;
  99   auto SaveExecInst = MRI.getUniqueVRegDef(SavedExec);
 100   if (!SaveExecInst || !isFullExecCopy(*SaveExecInst))
 101     return nullptr;
 102   return SaveExecInst;
 103 }
 104
 105 // Optimize sequence
 106 //    %sel = V_CNDMASK_B32_e64 0, 1, %cc
 107 //    %cmp = V_CMP_NE_U32 1, %1
 108 //    $vcc = S_AND_B64 $exec, %cmp
 109 //    S_CBRANCH_VCC[N]Z
 110 // =>
 111 //    $vcc = S_ANDN2_B64 $exec, %cc
 112 //    S_CBRANCH_VCC[N]Z
 113 //
 114 // It is the negation pattern inserted by DAGCombiner::visitBRCOND() in the
 115 // rebuildSetCC(). We start with S_CBRANCH to avoid exhaustive search, but
 116 // only 3 first instructions are really needed. S_AND_B64 with exec is a
 117 // required part of the pattern since V_CNDMASK_B32 writes zeroes for inactive
 118 // lanes.
 119 //
 120 // Returns %cc register on success.
 121 static unsigned optimizeVcndVcmpPair(MachineBasicBlock &MBB,
 122                                      const GCNSubtarget &ST,
 123                                      MachineRegisterInfo &MRI,
 124                                      LiveIntervals *LIS) {
 125   const SIRegisterInfo *TRI = ST.getRegisterInfo();
 126   const SIInstrInfo *TII = ST.getInstrInfo();
 127   const unsigned AndOpc = AMDGPU::S_AND_B64;
 128   const unsigned Andn2Opc = AMDGPU::S_ANDN2_B64;
 129   const unsigned CondReg = AMDGPU::VCC;
 130   const unsigned ExecReg = AMDGPU::EXEC;
 131
 132   auto I = llvm::find_if(MBB.terminators(), [](const MachineInstr &MI) {
 133                            unsigned Opc = MI.getOpcode();
 134                            return Opc == AMDGPU::S_CBRANCH_VCCZ ||
 135                                   Opc == AMDGPU::S_CBRANCH_VCCNZ; });
 136   if (I == MBB.terminators().end())
 137     return AMDGPU::NoRegister;
 138
 139   auto *And = TRI->findReachingDef(CondReg, AMDGPU::NoSubRegister,
 140                                    *I, MRI, LIS);
 141   if (!And || And->getOpcode() != AndOpc ||
 142       !And->getOperand(1).isReg() || !And->getOperand(2).isReg())
 143     return AMDGPU::NoRegister;
 144
 145   MachineOperand *AndCC = &And->getOperand(1);
 146   unsigned CmpReg = AndCC->getReg();
 147   unsigned CmpSubReg = AndCC->getSubReg();
 148   if (CmpReg == ExecReg) {
 149     AndCC = &And->getOperand(2);
 150     CmpReg = AndCC->getReg();
 151     CmpSubReg = AndCC->getSubReg();
 152   } else if (And->getOperand(2).getReg() != ExecReg) {
 153     return AMDGPU::NoRegister;
 154   }
 155
 156   auto *Cmp = TRI->findReachingDef(CmpReg, CmpSubReg, *And, MRI, LIS);
 157   if (!Cmp || !(Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e32 ||
 158                 Cmp->getOpcode() == AMDGPU::V_CMP_NE_U32_e64) ||
 159       Cmp->getParent() != And->getParent())
 160     return AMDGPU::NoRegister;
 161
 162   MachineOperand *Op1 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src0);
 163   MachineOperand *Op2 = TII->getNamedOperand(*Cmp, AMDGPU::OpName::src1);
 164   if (Op1->isImm() && Op2->isReg())
 165     std::swap(Op1, Op2);
 166   if (!Op1->isReg() || !Op2->isImm() || Op2->getImm() != 1)
 167     return AMDGPU::NoRegister;
 168
 169   unsigned SelReg = Op1->getReg();
 170   auto *Sel = TRI->findReachingDef(SelReg, Op1->getSubReg(), *Cmp, MRI, LIS);
 171   if (!Sel || Sel->getOpcode() != AMDGPU::V_CNDMASK_B32_e64)
 172     return AMDGPU::NoRegister;
 173
 174   Op1 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src0);
 175   Op2 = TII->getNamedOperand(*Sel, AMDGPU::OpName::src1);
 176   MachineOperand *CC = TII->getNamedOperand(*Sel, AMDGPU::OpName::src2);
 177   if (!Op1->isImm() || !Op2->isImm() || !CC->isReg() ||
 178       Op1->getImm() != 0 || Op2->getImm() != 1)
 179     return AMDGPU::NoRegister;
 180
 181   LLVM_DEBUG(dbgs() << "Folding sequence:\n\t" << *Sel << '\t'
 182                     << *Cmp << '\t' << *And);
 183
 184   unsigned CCReg = CC->getReg();
 185   LIS->RemoveMachineInstrFromMaps(*And);
 186   MachineInstr *Andn2 = BuildMI(MBB, *And, And->getDebugLoc(),
 187                                 TII->get(Andn2Opc), And->getOperand(0).getReg())
 188                             .addReg(ExecReg)
 189                             .addReg(CCReg, CC->getSubReg());
 190   And->eraseFromParent();
 191   LIS->InsertMachineInstrInMaps(*Andn2);
 192
 193   LLVM_DEBUG(dbgs() << "=>\n\t" << *Andn2 << '\n');
 194
 195   // Try to remove compare. Cmp value should not used in between of cmp
 196   // and s_and_b64 if VCC or just unused if any other register.
 197   if ((TargetRegisterInfo::isVirtualRegister(CmpReg) &&
 198        MRI.use_nodbg_empty(CmpReg)) ||
 199       (CmpReg == CondReg &&
 200        std::none_of(std::next(Cmp->getIterator()), Andn2->getIterator(),
 201                     [&](const MachineInstr &MI) {
 202                       return MI.readsRegister(CondReg, TRI); }))) {
 203     LLVM_DEBUG(dbgs() << "Erasing: " << *Cmp << '\n');
 204
 205     LIS->RemoveMachineInstrFromMaps(*Cmp);
 206     Cmp->eraseFromParent();
 207
 208     // Try to remove v_cndmask_b32.
 209     if (TargetRegisterInfo::isVirtualRegister(SelReg) &&
 210         MRI.use_nodbg_empty(SelReg)) {
 211       LLVM_DEBUG(dbgs() << "Erasing: " << *Sel << '\n');
 212
 213       LIS->RemoveMachineInstrFromMaps(*Sel);
 214       Sel->eraseFromParent();
 215     }
 216   }
 217
 218   return CCReg;
 219 }
 220
 221 bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) {
 222   if (skipFunction(MF.getFunction()))
 223     return false;
 224
 225   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 226   const SIRegisterInfo *TRI = ST.getRegisterInfo();
 227   const SIInstrInfo *TII = ST.getInstrInfo();
 228   MachineRegisterInfo &MRI = MF.getRegInfo();
 229   LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
 230   DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI});
 231   bool Changed = false;
 232
 233   for (MachineBasicBlock &MBB : MF) {
 234
 235     if (unsigned Reg = optimizeVcndVcmpPair(MBB, ST, MRI, LIS)) {
 236       RecalcRegs.insert(Reg);
 237       RecalcRegs.insert(AMDGPU::VCC_LO);
 238       RecalcRegs.insert(AMDGPU::VCC_HI);
 239       RecalcRegs.insert(AMDGPU::SCC);
 240       Changed = true;
 241     }
 242
 243     // Try to remove unneeded instructions before s_endpgm.
 244     if (MBB.succ_empty()) {
 245       if (MBB.empty())
 246         continue;
 247
 248       // Skip this if the endpgm has any implicit uses, otherwise we would need
 249       // to be careful to update / remove them.
 250       MachineInstr &Term = MBB.back();
 251       if (Term.getOpcode() != AMDGPU::S_ENDPGM ||
 252           Term.getNumOperands() != 0)
 253         continue;
 254
 255       SmallVector<MachineBasicBlock*, 4> Blocks({&MBB});
 256
 257       while (!Blocks.empty()) {
 258         auto CurBB = Blocks.pop_back_val();
 259         auto I = CurBB->rbegin(), E = CurBB->rend();
 260         if (I != E) {
 261           if (I->isUnconditionalBranch() || I->getOpcode() == AMDGPU::S_ENDPGM)
 262             ++I;
 263           else if (I->isBranch())
 264             continue;
 265         }
 266
 267         while (I != E) {
 268           if (I->isDebugInstr()) {
 269             I = std::next(I);
 270             continue;
 271           }
 272
 273           if (I->mayStore() || I->isBarrier() || I->isCall() ||
 274               I->hasUnmodeledSideEffects() || I->hasOrderedMemoryRef())
 275             break;
 276
 277           LLVM_DEBUG(dbgs()
 278                      << "Removing no effect instruction: " << *I << '\n');
 279
 280           for (auto &Op : I->operands()) {
 281             if (Op.isReg())
 282               RecalcRegs.insert(Op.getReg());
 283           }
 284
 285           auto Next = std::next(I);
 286           LIS->RemoveMachineInstrFromMaps(*I);
 287           I->eraseFromParent();
 288           I = Next;
 289
 290           Changed = true;
 291         }
 292
 293         if (I != E)
 294           continue;
 295
 296         // Try to ascend predecessors.
 297         for (auto *Pred : CurBB->predecessors()) {
 298           if (Pred->succ_size() == 1)
 299             Blocks.push_back(Pred);
 300         }
 301       }
 302       continue;
 303     }
 304
 305     // Try to collapse adjacent endifs.
 306     auto Lead = MBB.begin(), E = MBB.end();
 307     if (MBB.succ_size() != 1 || Lead == E || !isEndCF(*Lead, TRI))
 308       continue;
 309
 310     const MachineBasicBlock* Succ = *MBB.succ_begin();
 311     if (!MBB.isLayoutSuccessor(Succ))
 312       continue;
 313
 314     auto I = std::next(Lead);
 315
 316     for ( ; I != E; ++I)
 317       if (!TII->isSALU(*I) || I->readsRegister(AMDGPU::EXEC, TRI))
 318         break;
 319
 320     if (I != E)
 321       continue;
 322
 323     const auto NextLead = Succ->begin();
 324     if (NextLead == Succ->end() || !isEndCF(*NextLead, TRI) ||
 325         !getOrExecSource(*NextLead, *TII, MRI))
 326       continue;
 327
 328     LLVM_DEBUG(dbgs() << "Redundant EXEC = S_OR_B64 found: " << *Lead << '\n');
 329
 330     auto SaveExec = getOrExecSource(*Lead, *TII, MRI);
 331     unsigned SaveExecReg = getOrNonExecReg(*Lead, *TII);
 332     for (auto &Op : Lead->operands()) {
 333       if (Op.isReg())
 334         RecalcRegs.insert(Op.getReg());
 335     }
 336
 337     LIS->RemoveMachineInstrFromMaps(*Lead);
 338     Lead->eraseFromParent();
 339     if (SaveExecReg) {
 340       LIS->removeInterval(SaveExecReg);
 341       LIS->createAndComputeVirtRegInterval(SaveExecReg);
 342     }
 343
 344     Changed = true;
 345
 346     // If the only use of saved exec in the removed instruction is S_AND_B64
 347     // fold the copy now.
 348     if (!SaveExec || !SaveExec->isFullCopy())
 349       continue;
 350
 351     unsigned SavedExec = SaveExec->getOperand(0).getReg();
 352     bool SafeToReplace = true;
 353     for (auto& U : MRI.use_nodbg_instructions(SavedExec)) {
 354       if (U.getParent() != SaveExec->getParent()) {
 355         SafeToReplace = false;
 356         break;
 357       }
 358
 359       LLVM_DEBUG(dbgs() << "Redundant EXEC COPY: " << *SaveExec << '\n');
 360     }
 361
 362     if (SafeToReplace) {
 363       LIS->RemoveMachineInstrFromMaps(*SaveExec);
 364       SaveExec->eraseFromParent();
 365       MRI.replaceRegWith(SavedExec, AMDGPU::EXEC);
 366       LIS->removeInterval(SavedExec);
 367     }
 368   }
 369
 370   if (Changed) {
 371     for (auto Reg : RecalcRegs) {
 372       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 373         LIS->removeInterval(Reg);
 374         if (!MRI.reg_empty(Reg))
 375           LIS->createAndComputeVirtRegInterval(Reg);
 376       } else {
 377         for (MCRegUnitIterator U(Reg, TRI); U.isValid(); ++U)
 378           LIS->removeRegUnit(*U);
 379       }
 380     }
 381   }
 382
 383   return Changed;
 384 }