lib/Target/AMDGPU/SIFormMemoryClauses.cpp

   1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This pass creates bundles of SMEM and VMEM instructions forming memory
  11 /// clauses if XNACK is enabled. Def operands of clauses are marked as early
  12 /// clobber to make sure we will not override any source within a clause.
  13 ///
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "AMDGPU.h"
  17 #include "AMDGPUSubtarget.h"
  18 #include "GCNRegPressure.h"
  19 #include "SIInstrInfo.h"
  20 #include "SIMachineFunctionInfo.h"
  21 #include "SIRegisterInfo.h"
  22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  23 #include "llvm/ADT/DenseMap.h"
  24 #include "llvm/CodeGen/LiveIntervals.h"
  25 #include "llvm/CodeGen/MachineFunctionPass.h"
  26
  27 using namespace llvm;
  28
  29 #define DEBUG_TYPE "si-form-memory-clauses"
  30
  31 // Clauses longer then 15 instructions would overflow one of the counters
  32 // and stall. They can stall even earlier if there are outstanding counters.
  33 static cl::opt<unsigned>
  34 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
  35           cl::desc("Maximum length of a memory clause, instructions"));
  36
  37 namespace {
  38
  39 class SIFormMemoryClauses : public MachineFunctionPass {
  40   typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
  41
  42 public:
  43   static char ID;
  44
  45 public:
  46   SIFormMemoryClauses() : MachineFunctionPass(ID) {
  47     initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
  48   }
  49
  50   bool runOnMachineFunction(MachineFunction &MF) override;
  51
  52   StringRef getPassName() const override {
  53     return "SI Form memory clauses";
  54   }
  55
  56   void getAnalysisUsage(AnalysisUsage &AU) const override {
  57     AU.addRequired<LiveIntervals>();
  58     AU.setPreservesAll();
  59     MachineFunctionPass::getAnalysisUsage(AU);
  60   }
  61
  62 private:
  63   template <typename Callable>
  64   void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
  65
  66   bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
  67   bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
  68   void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
  69   bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
  70                       GCNDownwardRPTracker &RPT);
  71
  72   const GCNSubtarget *ST;
  73   const SIRegisterInfo *TRI;
  74   const MachineRegisterInfo *MRI;
  75   SIMachineFunctionInfo *MFI;
  76
  77   unsigned LastRecordedOccupancy;
  78   unsigned MaxVGPRs;
  79   unsigned MaxSGPRs;
  80 };
  81
  82 } // End anonymous namespace.
  83
  84 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
  85                       "SI Form memory clauses", false, false)
  86 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
  87 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
  88                     "SI Form memory clauses", false, false)
  89
  90
  91 char SIFormMemoryClauses::ID = 0;
  92
  93 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
  94
  95 FunctionPass *llvm::createSIFormMemoryClausesPass() {
  96   return new SIFormMemoryClauses();
  97 }
  98
  99 static bool isVMEMClauseInst(const MachineInstr &MI) {
 100   return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
 101 }
 102
 103 static bool isSMEMClauseInst(const MachineInstr &MI) {
 104   return SIInstrInfo::isSMRD(MI);
 105 }
 106
 107 // There no sense to create store clauses, they do not define anything,
 108 // thus there is nothing to set early-clobber.
 109 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
 110   if (MI.isDebugValue() || MI.isBundled())
 111     return false;
 112   if (!MI.mayLoad() || MI.mayStore())
 113     return false;
 114   if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
 115       AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
 116     return false;
 117   if (IsVMEMClause && !isVMEMClauseInst(MI))
 118     return false;
 119   if (!IsVMEMClause && !isSMEMClauseInst(MI))
 120     return false;
 121   // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
 122   for (const MachineOperand &ResMO : MI.defs()) {
 123     Register ResReg = ResMO.getReg();
 124     for (const MachineOperand &MO : MI.uses()) {
 125       if (!MO.isReg() || MO.isDef())
 126         continue;
 127       if (MO.getReg() == ResReg)
 128         return false;
 129     }
 130     break; // Only check the first def.
 131   }
 132   return true;
 133 }
 134
 135 static unsigned getMopState(const MachineOperand &MO) {
 136   unsigned S = 0;
 137   if (MO.isImplicit())
 138     S |= RegState::Implicit;
 139   if (MO.isDead())
 140     S |= RegState::Dead;
 141   if (MO.isUndef())
 142     S |= RegState::Undef;
 143   if (MO.isKill())
 144     S |= RegState::Kill;
 145   if (MO.isEarlyClobber())
 146     S |= RegState::EarlyClobber;
 147   if (Register::isPhysicalRegister(MO.getReg()) && MO.isRenamable())
 148     S |= RegState::Renamable;
 149   return S;
 150 }
 151
 152 template <typename Callable>
 153 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
 154                                       Callable Func) const {
 155   if (LaneMask.all() || Register::isPhysicalRegister(Reg) ||
 156       LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
 157     Func(0);
 158     return;
 159   }
 160
 161   const TargetRegisterClass *RC = MRI->getRegClass(Reg);
 162   unsigned E = TRI->getNumSubRegIndices();
 163   SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
 164   for (unsigned Idx = 1; Idx < E; ++Idx) {
 165     // Is this index even compatible with the given class?
 166     if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
 167       continue;
 168     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
 169     // Early exit if we found a perfect match.
 170     if (SubRegMask == LaneMask) {
 171       Func(Idx);
 172       return;
 173     }
 174
 175     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
 176       continue;
 177
 178     CoveringSubregs.push_back(Idx);
 179   }
 180
 181   llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
 182     LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
 183     LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
 184     unsigned NA = MaskA.getNumLanes();
 185     unsigned NB = MaskB.getNumLanes();
 186     if (NA != NB)
 187       return NA > NB;
 188     return MaskA.getHighestLane() > MaskB.getHighestLane();
 189   });
 190
 191   for (unsigned Idx : CoveringSubregs) {
 192     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
 193     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
 194       continue;
 195
 196     Func(Idx);
 197     LaneMask &= ~SubRegMask;
 198     if (LaneMask.none())
 199       return;
 200   }
 201
 202   llvm_unreachable("Failed to find all subregs to cover lane mask");
 203 }
 204
 205 // Returns false if there is a use of a def already in the map.
 206 // In this case we must break the clause.
 207 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
 208                                     RegUse &Defs, RegUse &Uses) const {
 209   // Check interference with defs.
 210   for (const MachineOperand &MO : MI.operands()) {
 211     // TODO: Prologue/Epilogue Insertion pass does not process bundled
 212     //       instructions.
 213     if (MO.isFI())
 214       return false;
 215
 216     if (!MO.isReg())
 217       continue;
 218
 219     Register Reg = MO.getReg();
 220
 221     // If it is tied we will need to write same register as we read.
 222     if (MO.isTied())
 223       return false;
 224
 225     RegUse &Map = MO.isDef() ? Uses : Defs;
 226     auto Conflict = Map.find(Reg);
 227     if (Conflict == Map.end())
 228       continue;
 229
 230     if (Register::isPhysicalRegister(Reg))
 231       return false;
 232
 233     LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
 234     if ((Conflict->second.second & Mask).any())
 235       return false;
 236   }
 237
 238   return true;
 239 }
 240
 241 // Since all defs in the clause are early clobber we can run out of registers.
 242 // Function returns false if pressure would hit the limit if instruction is
 243 // bundled into a memory clause.
 244 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
 245                                         GCNDownwardRPTracker &RPT) {
 246   // NB: skip advanceBeforeNext() call. Since all defs will be marked
 247   // early-clobber they will all stay alive at least to the end of the
 248   // clause. Therefor we should not decrease pressure even if load
 249   // pointer becomes dead and could otherwise be reused for destination.
 250   RPT.advanceToNext();
 251   GCNRegPressure MaxPressure = RPT.moveMaxPressure();
 252   unsigned Occupancy = MaxPressure.getOccupancy(*ST);
 253   if (Occupancy >= MFI->getMinAllowedOccupancy() &&
 254       MaxPressure.getVGPRNum() <= MaxVGPRs &&
 255       MaxPressure.getSGPRNum() <= MaxSGPRs) {
 256     LastRecordedOccupancy = Occupancy;
 257     return true;
 258   }
 259   return false;
 260 }
 261
 262 // Collect register defs and uses along with their lane masks and states.
 263 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
 264                                          RegUse &Defs, RegUse &Uses) const {
 265   for (const MachineOperand &MO : MI.operands()) {
 266     if (!MO.isReg())
 267       continue;
 268     Register Reg = MO.getReg();
 269     if (!Reg)
 270       continue;
 271
 272     LaneBitmask Mask = Register::isVirtualRegister(Reg)
 273                            ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
 274                            : LaneBitmask::getAll();
 275     RegUse &Map = MO.isDef() ? Defs : Uses;
 276
 277     auto Loc = Map.find(Reg);
 278     unsigned State = getMopState(MO);
 279     if (Loc == Map.end()) {
 280       Map[Reg] = std::make_pair(State, Mask);
 281     } else {
 282       Loc->second.first |= State;
 283       Loc->second.second |= Mask;
 284     }
 285   }
 286 }
 287
 288 // Check register def/use conflicts, occupancy limits and collect def/use maps.
 289 // Return true if instruction can be bundled with previous. It it cannot
 290 // def/use maps are not updated.
 291 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
 292                                          RegUse &Defs, RegUse &Uses,
 293                                          GCNDownwardRPTracker &RPT) {
 294   if (!canBundle(MI, Defs, Uses))
 295     return false;
 296
 297   if (!checkPressure(MI, RPT))
 298     return false;
 299
 300   collectRegUses(MI, Defs, Uses);
 301   return true;
 302 }
 303
 304 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
 305   if (skipFunction(MF.getFunction()))
 306     return false;
 307
 308   ST = &MF.getSubtarget<GCNSubtarget>();
 309   if (!ST->isXNACKEnabled())
 310     return false;
 311
 312   const SIInstrInfo *TII = ST->getInstrInfo();
 313   TRI = ST->getRegisterInfo();
 314   MRI = &MF.getRegInfo();
 315   MFI = MF.getInfo<SIMachineFunctionInfo>();
 316   LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
 317   SlotIndexes *Ind = LIS->getSlotIndexes();
 318   bool Changed = false;
 319
 320   MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
 321   MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
 322   unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
 323       MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);
 324
 325   for (MachineBasicBlock &MBB : MF) {
 326     MachineBasicBlock::instr_iterator Next;
 327     for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
 328       MachineInstr &MI = *I;
 329       Next = std::next(I);
 330
 331       bool IsVMEM = isVMEMClauseInst(MI);
 332
 333       if (!isValidClauseInst(MI, IsVMEM))
 334         continue;
 335
 336       RegUse Defs, Uses;
 337       GCNDownwardRPTracker RPT(*LIS);
 338       RPT.reset(MI);
 339
 340       if (!processRegUses(MI, Defs, Uses, RPT))
 341         continue;
 342
 343       unsigned Length = 1;
 344       for ( ; Next != E && Length < FuncMaxClause; ++Next) {
 345         if (!isValidClauseInst(*Next, IsVMEM))
 346           break;
 347
 348         // A load from pointer which was loaded inside the same bundle is an
 349         // impossible clause because we will need to write and read the same
 350         // register inside. In this case processRegUses will return false.
 351         if (!processRegUses(*Next, Defs, Uses, RPT))
 352           break;
 353
 354         ++Length;
 355       }
 356       if (Length < 2)
 357         continue;
 358
 359       Changed = true;
 360       MFI->limitOccupancy(LastRecordedOccupancy);
 361
 362       auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
 363       Ind->insertMachineInstrInMaps(*B);
 364
 365       for (auto BI = I; BI != Next; ++BI) {
 366         BI->bundleWithPred();
 367         Ind->removeSingleMachineInstrFromMaps(*BI);
 368
 369         for (MachineOperand &MO : BI->defs())
 370           if (MO.readsReg())
 371             MO.setIsInternalRead(true);
 372       }
 373
 374       for (auto &&R : Defs) {
 375         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
 376           unsigned S = R.second.first | RegState::EarlyClobber;
 377           if (!SubReg)
 378             S &= ~(RegState::Undef | RegState::Dead);
 379           B.addDef(R.first, S, SubReg);
 380         });
 381       }
 382
 383       for (auto &&R : Uses) {
 384         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
 385           B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
 386         });
 387       }
 388
 389       for (auto &&R : Defs) {
 390         unsigned Reg = R.first;
 391         Uses.erase(Reg);
 392         if (Register::isPhysicalRegister(Reg))
 393           continue;
 394         LIS->removeInterval(Reg);
 395         LIS->createAndComputeVirtRegInterval(Reg);
 396       }
 397
 398       for (auto &&R : Uses) {
 399         unsigned Reg = R.first;
 400         if (Register::isPhysicalRegister(Reg))
 401           continue;
 402         LIS->removeInterval(Reg);
 403         LIS->createAndComputeVirtRegInterval(Reg);
 404       }
 405     }
 406   }
 407
 408   return Changed;
 409 }