llvm/lib/Target/AMDGPU/GCNRegPressure.h

   1 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file defines the GCNRegPressure class, which tracks registry pressure
  11 /// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It
  12 /// also implements a compare function, which compares different register
  13 /// pressures, and declares one with max occupance as winner.
  14 ///
  15 //===----------------------------------------------------------------------===//
  16
  17 #ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
  18 #define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H
  19
  20 #include "GCNSubtarget.h"
  21 #include "llvm/CodeGen/LiveIntervals.h"
  22 #include <algorithm>
  23
  24 namespace llvm {
  25
  26 class MachineRegisterInfo;
  27 class raw_ostream;
  28 class SlotIndex;
  29
  30 struct GCNRegPressure {
  31   enum RegKind {
  32     SGPR32,
  33     SGPR_TUPLE,
  34     VGPR32,
  35     VGPR_TUPLE,
  36     AGPR32,
  37     AGPR_TUPLE,
  38     TOTAL_KINDS
  39   };
  40
  41   GCNRegPressure() {
  42     clear();
  43   }
  44
  45   bool empty() const { return getSGPRNum() == 0 && getVGPRNum(false) == 0; }
  46
  47   void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
  48
  49   unsigned getSGPRNum() const { return Value[SGPR32]; }
  50   unsigned getVGPRNum(bool UnifiedVGPRFile) const {
  51     if (UnifiedVGPRFile) {
  52       return Value[AGPR32] ? alignTo(Value[VGPR32], 4) + Value[AGPR32]
  53                            : Value[VGPR32] + Value[AGPR32];
  54     }
  55     return std::max(Value[VGPR32], Value[AGPR32]);
  56   }
  57   unsigned getAGPRNum() const { return Value[AGPR32]; }
  58
  59   unsigned getVGPRTuplesWeight() const { return std::max(Value[VGPR_TUPLE],
  60                                                          Value[AGPR_TUPLE]); }
  61   unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
  62
  63   unsigned getOccupancy(const GCNSubtarget &ST) const {
  64     return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
  65              ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts())));
  66   }
  67
  68   void inc(unsigned Reg,
  69            LaneBitmask PrevMask,
  70            LaneBitmask NewMask,
  71            const MachineRegisterInfo &MRI);
  72
  73   bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const {
  74     return getOccupancy(ST) > O.getOccupancy(ST);
  75   }
  76
  77   bool less(const GCNSubtarget &ST, const GCNRegPressure& O,
  78     unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
  79
  80   bool operator==(const GCNRegPressure &O) const {
  81     return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
  82   }
  83
  84   bool operator!=(const GCNRegPressure &O) const {
  85     return !(*this == O);
  86   }
  87
  88   void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const;
  89   void dump() const { print(dbgs()); }
  90
  91 private:
  92   unsigned Value[TOTAL_KINDS];
  93
  94   static unsigned getRegKind(Register Reg, const MachineRegisterInfo &MRI);
  95
  96   friend GCNRegPressure max(const GCNRegPressure &P1,
  97                             const GCNRegPressure &P2);
  98 };
  99
 100 inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {
 101   GCNRegPressure Res;
 102   for (unsigned I = 0; I < GCNRegPressure::TOTAL_KINDS; ++I)
 103     Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);
 104   return Res;
 105 }
 106
 107 class GCNRPTracker {
 108 public:
 109   using LiveRegSet = DenseMap<unsigned, LaneBitmask>;
 110
 111 protected:
 112   const LiveIntervals &LIS;
 113   LiveRegSet LiveRegs;
 114   GCNRegPressure CurPressure, MaxPressure;
 115   const MachineInstr *LastTrackedMI = nullptr;
 116   mutable const MachineRegisterInfo *MRI = nullptr;
 117
 118   GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}
 119
 120   void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,
 121              bool After);
 122
 123 public:
 124   // live regs for the current state
 125   const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }
 126   const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }
 127
 128   void clearMaxPressure() { MaxPressure.clear(); }
 129
 130   // returns MaxPressure, resetting it
 131   decltype(MaxPressure) moveMaxPressure() {
 132     auto Res = MaxPressure;
 133     MaxPressure.clear();
 134     return Res;
 135   }
 136
 137   decltype(LiveRegs) moveLiveRegs() {
 138     return std::move(LiveRegs);
 139   }
 140
 141   static void printLiveRegs(raw_ostream &OS, const LiveRegSet& LiveRegs,
 142                             const MachineRegisterInfo &MRI);
 143 };
 144
 145 class GCNUpwardRPTracker : public GCNRPTracker {
 146 public:
 147   GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
 148
 149   // reset tracker to the point just below MI
 150   // filling live regs upon this point using LIS
 151   void reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
 152
 153   // move to the state just above the MI
 154   void recede(const MachineInstr &MI);
 155
 156   // checks whether the tracker's state after receding MI corresponds
 157   // to reported by LIS
 158   bool isValid() const;
 159 };
 160
 161 class GCNDownwardRPTracker : public GCNRPTracker {
 162   // Last position of reset or advanceBeforeNext
 163   MachineBasicBlock::const_iterator NextMI;
 164
 165   MachineBasicBlock::const_iterator MBBEnd;
 166
 167 public:
 168   GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}
 169
 170   MachineBasicBlock::const_iterator getNext() const { return NextMI; }
 171
 172   // Reset tracker to the point before the MI
 173   // filling live regs upon this point using LIS.
 174   // Returns false if block is empty except debug values.
 175   bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);
 176
 177   // Move to the state right before the next MI. Returns false if reached
 178   // end of the block.
 179   bool advanceBeforeNext();
 180
 181   // Move to the state at the MI, advanceBeforeNext has to be called first.
 182   void advanceToNext();
 183
 184   // Move to the state at the next MI. Returns false if reached end of block.
 185   bool advance();
 186
 187   // Advance instructions until before End.
 188   bool advance(MachineBasicBlock::const_iterator End);
 189
 190   // Reset to Begin and advance to End.
 191   bool advance(MachineBasicBlock::const_iterator Begin,
 192                MachineBasicBlock::const_iterator End,
 193                const LiveRegSet *LiveRegsCopy = nullptr);
 194 };
 195
 196 LaneBitmask getLiveLaneMask(unsigned Reg,
 197                             SlotIndex SI,
 198                             const LiveIntervals &LIS,
 199                             const MachineRegisterInfo &MRI);
 200
 201 GCNRPTracker::LiveRegSet getLiveRegs(SlotIndex SI,
 202                                      const LiveIntervals &LIS,
 203                                      const MachineRegisterInfo &MRI);
 204
 205 /// creates a map MachineInstr -> LiveRegSet
 206 /// R - range of iterators on instructions
 207 /// After - upon entry or exit of every instruction
 208 /// Note: there is no entry in the map for instructions with empty live reg set
 209 /// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))
 210 template <typename Range>
 211 DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>
 212 getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
 213   std::vector<SlotIndex> Indexes;
 214   Indexes.reserve(std::distance(R.begin(), R.end()));
 215   auto &SII = *LIS.getSlotIndexes();
 216   for (MachineInstr *I : R) {
 217     auto SI = SII.getInstructionIndex(*I);
 218     Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());
 219   }
 220   llvm::sort(Indexes);
 221
 222   auto &MRI = (*R.begin())->getParent()->getParent()->getRegInfo();
 223   DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;
 224   SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;
 225   for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
 226     auto Reg = Register::index2VirtReg(I);
 227     if (!LIS.hasInterval(Reg))
 228       continue;
 229     auto &LI = LIS.getInterval(Reg);
 230     LiveIdxs.clear();
 231     if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs)))
 232       continue;
 233     if (!LI.hasSubRanges()) {
 234       for (auto SI : LiveIdxs)
 235         LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =
 236           MRI.getMaxLaneMaskForVReg(Reg);
 237     } else
 238       for (const auto &S : LI.subranges()) {
 239         // constrain search for subranges by indexes live at main range
 240         SRLiveIdxs.clear();
 241         S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs));
 242         for (auto SI : SRLiveIdxs)
 243           LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask;
 244       }
 245   }
 246   return LiveRegMap;
 247 }
 248
 249 inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,
 250                                                  const LiveIntervals &LIS) {
 251   return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS,
 252                      MI.getParent()->getParent()->getRegInfo());
 253 }
 254
 255 inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,
 256                                                   const LiveIntervals &LIS) {
 257   return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS,
 258                      MI.getParent()->getParent()->getRegInfo());
 259 }
 260
 261 template <typename Range>
 262 GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,
 263                               Range &&LiveRegs) {
 264   GCNRegPressure Res;
 265   for (const auto &RM : LiveRegs)
 266     Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI);
 267   return Res;
 268 }
 269
 270 bool isEqual(const GCNRPTracker::LiveRegSet &S1,
 271              const GCNRPTracker::LiveRegSet &S2);
 272
 273 void printLivesAt(SlotIndex SI,
 274                   const LiveIntervals &LIS,
 275                   const MachineRegisterInfo &MRI);
 276
 277 } // end namespace llvm
 278
 279 #endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H