llvm/lib/Target/AMDGPU/AMDGPUMarkLastScratchLoad.cpp

   1 //===-- AMDGPUMarkLastScratchLoad.cpp -------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Mark scratch load/spill instructions which are guaranteed to be the last time
  10 // this scratch slot is used so it can be evicted from caches.
  11 //
  12 // TODO: Handle general stack accesses not just spilling.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "AMDGPU.h"
  17 #include "GCNSubtarget.h"
  18 #include "llvm/CodeGen/LiveIntervals.h"
  19 #include "llvm/CodeGen/LiveStacks.h"
  20 #include "llvm/CodeGen/MachineOperand.h"
  21
  22 using namespace llvm;
  23
  24 #define DEBUG_TYPE "amdgpu-mark-last-scratch-load"
  25
  26 namespace {
  27
  28 class AMDGPUMarkLastScratchLoad : public MachineFunctionPass {
  29 private:
  30   LiveStacks *LS = nullptr;
  31   LiveIntervals *LIS = nullptr;
  32   SlotIndexes *SI = nullptr;
  33   const SIInstrInfo *SII = nullptr;
  34
  35 public:
  36   static char ID;
  37
  38   AMDGPUMarkLastScratchLoad() : MachineFunctionPass(ID) {
  39     initializeAMDGPUMarkLastScratchLoadPass(*PassRegistry::getPassRegistry());
  40   }
  41
  42   bool runOnMachineFunction(MachineFunction &MF) override;
  43
  44   void getAnalysisUsage(AnalysisUsage &AU) const override {
  45     AU.addRequired<SlotIndexesWrapperPass>();
  46     AU.addRequired<LiveIntervalsWrapperPass>();
  47     AU.addRequired<LiveStacks>();
  48     AU.setPreservesAll();
  49     MachineFunctionPass::getAnalysisUsage(AU);
  50   }
  51
  52   StringRef getPassName() const override {
  53     return "AMDGPU Mark Last Scratch Load";
  54   }
  55 };
  56
  57 } // end anonymous namespace
  58
  59 bool AMDGPUMarkLastScratchLoad::runOnMachineFunction(MachineFunction &MF) {
  60   if (skipFunction(MF.getFunction()))
  61     return false;
  62
  63   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  64   if (ST.getGeneration() < AMDGPUSubtarget::GFX12)
  65     return false;
  66
  67   LS = &getAnalysis<LiveStacks>();
  68   LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
  69   SI = &getAnalysis<SlotIndexesWrapperPass>().getSI();
  70   SII = ST.getInstrInfo();
  71   SlotIndexes &Slots = *LIS->getSlotIndexes();
  72
  73   const unsigned NumSlots = LS->getNumIntervals();
  74   if (NumSlots == 0) {
  75     LLVM_DEBUG(dbgs() << "No live slots, skipping\n");
  76     return false;
  77   }
  78
  79   LLVM_DEBUG(dbgs() << LS->getNumIntervals() << " intervals\n");
  80
  81   bool Changed = false;
  82
  83   for (auto &[SS, LI] : *LS) {
  84     for (const LiveRange::Segment &Segment : LI.segments) {
  85
  86       // Ignore segments that run to the end of basic block because in this case
  87       // slot is still live at the end of it.
  88       if (Segment.end.isBlock())
  89         continue;
  90
  91       const int FrameIndex = Register::stackSlot2Index(LI.reg());
  92       MachineInstr *LastLoad = nullptr;
  93
  94       MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(Segment.end);
  95
  96       // If there is no instruction at this slot because it was deleted take the
  97       // instruction from the next slot.
  98       if (!MISegmentEnd) {
  99         SlotIndex NextSlot = Slots.getNextNonNullIndex(Segment.end);
 100         MISegmentEnd = SI->getInstructionFromIndex(NextSlot);
 101       }
 102
 103       MachineInstr *MISegmentStart = SI->getInstructionFromIndex(Segment.start);
 104       MachineBasicBlock *BB = MISegmentEnd->getParent();
 105
 106       // Start iteration backwards from segment end until the start of basic
 107       // block or start of segment if it is in the same basic block.
 108       auto End = BB->rend();
 109       if (MISegmentStart && MISegmentStart->getParent() == BB)
 110         End = MISegmentStart->getReverseIterator();
 111
 112       for (auto MI = MISegmentEnd->getReverseIterator(); MI != End; ++MI) {
 113         int LoadFI = 0;
 114
 115         if (SII->isLoadFromStackSlot(*MI, LoadFI) && LoadFI == FrameIndex) {
 116           LastLoad = &*MI;
 117           break;
 118         }
 119       }
 120
 121       if (LastLoad && !LastLoad->memoperands_empty()) {
 122         MachineMemOperand *MMO = *LastLoad->memoperands_begin();
 123         MMO->setFlags(MOLastUse);
 124         Changed = true;
 125         LLVM_DEBUG(dbgs() << "  Found last load: " << *LastLoad);
 126       }
 127     }
 128   }
 129
 130   return Changed;
 131 }
 132
 133 char AMDGPUMarkLastScratchLoad::ID = 0;
 134
 135 char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoad::ID;
 136
 137 INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoad, DEBUG_TYPE,
 138                       "AMDGPU Mark last scratch load", false, false)
 139 INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
 140 INITIALIZE_PASS_DEPENDENCY(LiveStacks)
 141 INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoad, DEBUG_TYPE,
 142                     "AMDGPU Mark last scratch load", false, false)