lib/Target/AMDGPU/GCNRegPressure.h

   1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
  10 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
  11
  12 #include "AMDGPUSubtarget.h"
  13 #include "llvm/ADT/DenseMap.h"
  14 #include "llvm/CodeGen/LiveIntervals.h"
  15 #include "llvm/CodeGen/MachineBasicBlock.h"
  16 #include "llvm/CodeGen/MachineInstr.h"
  17 #include "llvm/CodeGen/SlotIndexes.h"
  18 #include "llvm/MC/LaneBitmask.h"
  19 #include "llvm/Support/Debug.h"
  20 #include <algorithm>
  21 #include <limits>
  22
  23 namespace llvm {
  24
  25 class MachineRegisterInfo;
  26 class raw_ostream;
  27
  28 struct GCNRegPressure {
  29   enum RegKind {
  30     SGPR32,
  31     SGPR_TUPLE,
  32     VGPR32,
  33     VGPR_TUPLE,
  34     AGPR32,
  35     AGPR_TUPLE,
  36     TOTAL_KINDS
  37   };
  38
  39   GCNRegPressure() {
  40     clear();
  41   }
  42
  43   bool empty() const { return getSGPRNum() == 0 && getVGPRNum() == 0; }
  44
  45   void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
  46
  47   unsigned getSGPRNum() const { return Value[SGPR32]; }
  48   unsigned getVGPRNum() const { return std::max(Value[VGPR32], Value[AGPR32]); }
  49
  50   unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
  51                                                          Value[AGPR_TUPLE]); }
  52   unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
  53
  54   unsigned getOccupancy(const GCNSubtarget &ST) const {
  55     return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
  56                     ST.getOccupancyWithNumVGPRs(getVGPRNum()));
  57   }
  58
  59   void inc(unsigned Reg,
  60            LaneBitmask PrevMask,
  61            LaneBitmask NewMask,
  62            const MachineRegisterInfo &MRI);
  63
  64   bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const {
  65     return getOccupancy(ST) > O.getOccupancy(ST);
  66   }
  67
  68   bool less(const GCNSubtarget &ST, const GCNRegPressure& O,
  69     unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
  70
  71   bool operator==(const GCNRegPressure &O) const {
  72     return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
  73   }
  74
  75   bool operator!=(const GCNRegPressure &O) const {
  76     return !(*this == O);
  77   }
  78
  79   void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const;
  80   void dump() const { print(dbgs()); }
  81
  82 private:
  83   unsigned Value[TOTAL_KINDS];
  84
  85   static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI);
  86
  87   friend GCNRegPressure max(const GCNRegPressure &P1,
  88                             const GCNRegPressure &P2);
  89 };
  90
  91 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {
  92   GCNRegPressure Res;
  93   for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I)
  94     Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);
  95   return Res;
  96 }
  97
  98 class GCNRPTracker {
  99 public:
 100   using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
 101
 102 protected:
 103   const LiveIntervals &LIS;
 104   LiveRegSet LiveRegs;
 105   GCNRegPressure CurPressure, MaxPressure;
 106   const MachineInstr *LastTrackedMI = nullptr;
 107   mutable const MachineRegisterInfo *MRI = nullptr;
 108
 109   GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
 110
 111   void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,
 112              bool After);
 113
 114 public:
 115   // live regs for the current state
 116   const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
 117   const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
 118
 119   void clearMaxPressure() { MaxPressure.clear(); }
 120
 121   // returns MaxPressure, resetting it
 122   decltype(MaxPressure) moveMaxPressure() {
 123     auto Res = MaxPressure;
 124     MaxPressure.clear();
 125     return Res;
 126   }
 127
 128   decltype(LiveRegs) moveLiveRegs() {
 129     return std::move(LiveRegs);
 130   }
 131
 132   static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs,
 133                             const MachineRegisterInfo &MRI);
 134 };
 135
 136 class GCNUpwardRPTracker : public GCNRPTracker {
 137 public:
 138   GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
 139
 140   // reset tracker to the point just below MI
 141   // filling live regs upon this point using LIS
 142   void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
 143
 144   // move to the state just above the MI
 145   void recede(const MachineInstr &MI);
 146
 147   // checks whether the tracker's state after receding MI corresponds
 148   // to reported by LIS
 149   bool isValid() const;
 150 };
 151
 152 class GCNDownwardRPTracker : public GCNRPTracker {
 153   // Last position of reset or advanceBeforeNext
 154   MachineBasicBlock::const_iterator NextMI;
 155
 156   MachineBasicBlock::const_iterator MBBEnd;
 157
 158 public:
 159   GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
 160
 161   const MachineBasicBlock::const_iterator getNext() const { return NextMI; }
 162
 163   // Reset tracker to the point before the MI
 164   // filling live regs upon this point using LIS.
 165   // Returns false if block is empty except debug values.
 166   bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
 167
 168   // Move to the state right before the next MI. Returns false if reached
 169   // end of the block.
 170   bool advanceBeforeNext();
 171
 172   // Move to the state at the MI, advanceBeforeNext has to be called first.
 173   void advanceToNext();
 174
 175   // Move to the state at the next MI. Returns false if reached end of block.
 176   bool advance();
 177
 178   // Advance instructions until before End.
 179   bool advance(MachineBasicBlock::const_iterator End);
 180
 181   // Reset to Begin and advance to End.
 182   bool advance(MachineBasicBlock::const_iterator Begin,
 183                MachineBasicBlock::const_iterator End,
 184                const LiveRegSet *LiveRegsCopy = nullptr);
 185 };
 186
 187 LaneBitmask getLiveLaneMask(unsigned Reg,
 188                             SlotIndex SI,
 189                             const LiveIntervals &LIS,
 190                             const MachineRegisterInfo &MRI);
 191
 192 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI,
 193                                      const LiveIntervals &LIS,
 194                                      const MachineRegisterInfo &MRI);
 195
 196 /// creates a map MachineInstr -> LiveRegSet
 197 /// R - range of iterators on instructions
 198 /// After - upon entry or exit of every instruction
 199 /// Note: there is no entry in the map for instructions with empty live reg set
 200 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))
 201 template <typename Range>
 202 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>
 203 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
 204   std::vector<SlotIndex> Indexes;
 205   Indexes.reserve(std::distance(R.begin(), R.end()));
 206   auto &SII = *LIS.getSlotIndexes();
 207   for (MachineInstr *I : R) {
 208     auto SI = SII.getInstructionIndex(*I);
 209     Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());
 210   }
 211   std::sort(Indexes.begin(), Indexes.end());
 212
 213   auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo();
 214   DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
 215   SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;
 216   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
 217     auto Reg = Register::index2VirtReg(I);
 218     if (!LIS.hasInterval(Reg))
 219       continue;
 220     auto &LI = LIS.getInterval(Reg);
 221     LiveIdxs.clear();
 222     if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs)))
 223       continue;
 224     if (!LI.hasSubRanges()) {
 225       for (auto SI : LiveIdxs)
 226         LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =
 227           MRI.getMaxLaneMaskForVReg(Reg);
 228     } else
 229       for (const auto &S : LI.subranges()) {
 230         // constrain search for subranges by indexes live at main range
 231         SRLiveIdxs.clear();
 232         S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs));
 233         for (auto SI : SRLiveIdxs)
 234           LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask;
 235       }
 236   }
 237   return LiveRegMap;
 238 }
 239
 240 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,
 241                                                  const LiveIntervals &LIS) {
 242   return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS,
 243                      MI.getParent()->getParent()->getRegInfo());
 244 }
 245
 246 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,
 247                                                   const LiveIntervals &LIS) {
 248   return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS,
 249                      MI.getParent()->getParent()->getRegInfo());
 250 }
 251
 252 template <typename Range>
 253 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,
 254                               Range &&LiveRegs) {
 255   GCNRegPressure Res;
 256   for (const auto &RM : LiveRegs)
 257     Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI);
 258   return Res;
 259 }
 260
 261 bool isEqual(const GCNRPTracker::LiveRegSet &S1,
 262              const GCNRPTracker::LiveRegSet &S2);
 263
 264 void printLivesAt(SlotIndex SI,
 265                   const LiveIntervals &LIS,
 266                   const MachineRegisterInfo &MRI);
 267
 268 } // end namespace llvm
 269
 270 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H