lib/MCA/Stages/ExecuteStage.cpp

   1 //===---------------------- ExecuteStage.cpp --------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 ///
  10 /// This file defines the execution stage of an instruction pipeline.
  11 ///
  12 /// The ExecuteStage is responsible for managing the hardware scheduler
  13 /// and issuing notifications that an instruction has been executed.
  14 ///
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "llvm/MCA/Stages/ExecuteStage.h"
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/Support/Debug.h"
  20
  21 #define DEBUG_TYPE "llvm-mca"
  22
  23 namespace llvm {
  24 namespace mca {
  25
  26 HWStallEvent::GenericEventType toHWStallEventType(Scheduler::Status Status) {
  27   switch (Status) {
  28   case Scheduler::SC_LOAD_QUEUE_FULL:
  29     return HWStallEvent::LoadQueueFull;
  30   case Scheduler::SC_STORE_QUEUE_FULL:
  31     return HWStallEvent::StoreQueueFull;
  32   case Scheduler::SC_BUFFERS_FULL:
  33     return HWStallEvent::SchedulerQueueFull;
  34   case Scheduler::SC_DISPATCH_GROUP_STALL:
  35     return HWStallEvent::DispatchGroupStall;
  36   case Scheduler::SC_AVAILABLE:
  37     return HWStallEvent::Invalid;
  38   }
  39
  40   llvm_unreachable("Don't know how to process this StallKind!");
  41 }
  42
  43 bool ExecuteStage::isAvailable(const InstRef &IR) const {
  44   if (Scheduler::Status S = HWS.isAvailable(IR)) {
  45     HWStallEvent::GenericEventType ET = toHWStallEventType(S);
  46     notifyEvent<HWStallEvent>(HWStallEvent(ET, IR));
  47     return false;
  48   }
  49
  50   return true;
  51 }
  52
  53 Error ExecuteStage::issueInstruction(InstRef &IR) {
  54   SmallVector<std::pair<ResourceRef, ResourceCycles>, 4> Used;
  55   SmallVector<InstRef, 4> Pending;
  56   SmallVector<InstRef, 4> Ready;
  57
  58   HWS.issueInstruction(IR, Used, Pending, Ready);
  59   NumIssuedOpcodes += IR.getInstruction()->getDesc().NumMicroOps;
  60
  61   notifyReservedOrReleasedBuffers(IR, /* Reserved */ false);
  62
  63   notifyInstructionIssued(IR, Used);
  64   if (IR.getInstruction()->isExecuted()) {
  65     notifyInstructionExecuted(IR);
  66     // FIXME: add a buffer of executed instructions.
  67     if (Error S = moveToTheNextStage(IR))
  68       return S;
  69   }
  70
  71   for (const InstRef &I : Pending)
  72     notifyInstructionPending(I);
  73
  74   for (const InstRef &I : Ready)
  75     notifyInstructionReady(I);
  76   return ErrorSuccess();
  77 }
  78
  79 Error ExecuteStage::issueReadyInstructions() {
  80   InstRef IR = HWS.select();
  81   while (IR) {
  82     if (Error Err = issueInstruction(IR))
  83       return Err;
  84
  85     // Select the next instruction to issue.
  86     IR = HWS.select();
  87   }
  88
  89   return ErrorSuccess();
  90 }
  91
  92 Error ExecuteStage::cycleStart() {
  93   SmallVector<ResourceRef, 8> Freed;
  94   SmallVector<InstRef, 4> Executed;
  95   SmallVector<InstRef, 4> Pending;
  96   SmallVector<InstRef, 4> Ready;
  97
  98   HWS.cycleEvent(Freed, Executed, Pending, Ready);
  99   NumDispatchedOpcodes = 0;
 100   NumIssuedOpcodes = 0;
 101
 102   for (const ResourceRef &RR : Freed)
 103     notifyResourceAvailable(RR);
 104
 105   for (InstRef &IR : Executed) {
 106     notifyInstructionExecuted(IR);
 107     // FIXME: add a buffer of executed instructions.
 108     if (Error S = moveToTheNextStage(IR))
 109       return S;
 110   }
 111
 112   for (const InstRef &IR : Pending)
 113     notifyInstructionPending(IR);
 114
 115   for (const InstRef &IR : Ready)
 116     notifyInstructionReady(IR);
 117
 118   return issueReadyInstructions();
 119 }
 120
 121 Error ExecuteStage::cycleEnd() {
 122   if (!EnablePressureEvents)
 123     return ErrorSuccess();
 124
 125   // Always conservatively report any backpressure events if the dispatch logic
 126   // was stalled due to unavailable scheduler resources.
 127   if (!HWS.hadTokenStall() && NumDispatchedOpcodes <= NumIssuedOpcodes)
 128     return ErrorSuccess();
 129
 130   SmallVector<InstRef, 8> Insts;
 131   uint64_t Mask = HWS.analyzeResourcePressure(Insts);
 132   if (Mask) {
 133     LLVM_DEBUG(dbgs() << "[E] Backpressure increased because of unavailable "
 134                          "pipeline resources: "
 135                       << format_hex(Mask, 16) << '\n');
 136     HWPressureEvent Ev(HWPressureEvent::RESOURCES, Insts, Mask);
 137     notifyEvent(Ev);
 138   }
 139
 140   SmallVector<InstRef, 8> RegDeps;
 141   SmallVector<InstRef, 8> MemDeps;
 142   HWS.analyzeDataDependencies(RegDeps, MemDeps);
 143   if (RegDeps.size()) {
 144     LLVM_DEBUG(
 145         dbgs() << "[E] Backpressure increased by register dependencies\n");
 146     HWPressureEvent Ev(HWPressureEvent::REGISTER_DEPS, RegDeps);
 147     notifyEvent(Ev);
 148   }
 149
 150   if (MemDeps.size()) {
 151     LLVM_DEBUG(dbgs() << "[E] Backpressure increased by memory dependencies\n");
 152     HWPressureEvent Ev(HWPressureEvent::MEMORY_DEPS, MemDeps);
 153     notifyEvent(Ev);
 154   }
 155
 156   return ErrorSuccess();
 157 }
 158
 159 #ifndef NDEBUG
 160 static void verifyInstructionEliminated(const InstRef &IR) {
 161   const Instruction &Inst = *IR.getInstruction();
 162   assert(Inst.isEliminated() && "Instruction was not eliminated!");
 163   assert(Inst.isReady() && "Instruction in an inconsistent state!");
 164
 165   // Ensure that instructions eliminated at register renaming stage are in a
 166   // consistent state.
 167   const InstrDesc &Desc = Inst.getDesc();
 168   assert(!Desc.MayLoad && !Desc.MayStore && "Cannot eliminate a memory op!");
 169 }
 170 #endif
 171
 172 Error ExecuteStage::handleInstructionEliminated(InstRef &IR) {
 173 #ifndef NDEBUG
 174   verifyInstructionEliminated(IR);
 175 #endif
 176   notifyInstructionPending(IR);
 177   notifyInstructionReady(IR);
 178   notifyInstructionIssued(IR, {});
 179   IR.getInstruction()->forceExecuted();
 180   notifyInstructionExecuted(IR);
 181   return moveToTheNextStage(IR);
 182 }
 183
 184 // Schedule the instruction for execution on the hardware.
 185 Error ExecuteStage::execute(InstRef &IR) {
 186   assert(isAvailable(IR) && "Scheduler is not available!");
 187
 188 #ifndef NDEBUG
 189   // Ensure that the HWS has not stored this instruction in its queues.
 190   HWS.sanityCheck(IR);
 191 #endif
 192
 193   if (IR.getInstruction()->isEliminated())
 194     return handleInstructionEliminated(IR);
 195
 196   // Reserve a slot in each buffered resource. Also, mark units with
 197   // BufferSize=0 as reserved. Resources with a buffer size of zero will only
 198   // be released after MCIS is issued, and all the ResourceCycles for those
 199   // units have been consumed.
 200   bool IsReadyInstruction = HWS.dispatch(IR);
 201   const Instruction &Inst = *IR.getInstruction();
 202   NumDispatchedOpcodes += Inst.getDesc().NumMicroOps;
 203   notifyReservedOrReleasedBuffers(IR, /* Reserved */ true);
 204
 205   if (!IsReadyInstruction) {
 206     if (Inst.isPending())
 207       notifyInstructionPending(IR);
 208     return ErrorSuccess();
 209   }
 210
 211   notifyInstructionPending(IR);
 212
 213   // If we did not return early, then the scheduler is ready for execution.
 214   notifyInstructionReady(IR);
 215
 216   // If we cannot issue immediately, the HWS will add IR to its ready queue for
 217   // execution later, so we must return early here.
 218   if (!HWS.mustIssueImmediately(IR))
 219     return ErrorSuccess();
 220
 221   // Issue IR to the underlying pipelines.
 222   return issueInstruction(IR);
 223 }
 224
 225 void ExecuteStage::notifyInstructionExecuted(const InstRef &IR) const {
 226   LLVM_DEBUG(dbgs() << "[E] Instruction Executed: #" << IR << '\n');
 227   notifyEvent<HWInstructionEvent>(
 228       HWInstructionEvent(HWInstructionEvent::Executed, IR));
 229 }
 230
 231 void ExecuteStage::notifyInstructionPending(const InstRef &IR) const {
 232   LLVM_DEBUG(dbgs() << "[E] Instruction Pending: #" << IR << '\n');
 233   notifyEvent<HWInstructionEvent>(
 234       HWInstructionEvent(HWInstructionEvent::Pending, IR));
 235 }
 236
 237 void ExecuteStage::notifyInstructionReady(const InstRef &IR) const {
 238   LLVM_DEBUG(dbgs() << "[E] Instruction Ready: #" << IR << '\n');
 239   notifyEvent<HWInstructionEvent>(
 240       HWInstructionEvent(HWInstructionEvent::Ready, IR));
 241 }
 242
 243 void ExecuteStage::notifyResourceAvailable(const ResourceRef &RR) const {
 244   LLVM_DEBUG(dbgs() << "[E] Resource Available: [" << RR.first << '.'
 245                     << RR.second << "]\n");
 246   for (HWEventListener *Listener : getListeners())
 247     Listener->onResourceAvailable(RR);
 248 }
 249
 250 void ExecuteStage::notifyInstructionIssued(
 251     const InstRef &IR,
 252     MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const {
 253   LLVM_DEBUG({
 254     dbgs() << "[E] Instruction Issued: #" << IR << '\n';
 255     for (const std::pair<ResourceRef, ResourceCycles> &Resource : Used) {
 256       assert(Resource.second.getDenominator() == 1 && "Invalid cycles!");
 257       dbgs() << "[E] Resource Used: [" << Resource.first.first << '.'
 258              << Resource.first.second << "], ";
 259       dbgs() << "cycles: " << Resource.second.getNumerator() << '\n';
 260     }
 261   });
 262
 263   // Replace resource masks with valid resource processor IDs.
 264   for (std::pair<ResourceRef, ResourceCycles> &Use : Used)
 265     Use.first.first = HWS.getResourceID(Use.first.first);
 266
 267   notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, Used));
 268 }
 269
 270 void ExecuteStage::notifyReservedOrReleasedBuffers(const InstRef &IR,
 271                                                    bool Reserved) const {
 272   const InstrDesc &Desc = IR.getInstruction()->getDesc();
 273   if (Desc.Buffers.empty())
 274     return;
 275
 276   SmallVector<unsigned, 4> BufferIDs(Desc.Buffers.begin(), Desc.Buffers.end());
 277   std::transform(Desc.Buffers.begin(), Desc.Buffers.end(), BufferIDs.begin(),
 278                  [&](uint64_t Op) { return HWS.getResourceID(Op); });
 279   if (Reserved) {
 280     for (HWEventListener *Listener : getListeners())
 281       Listener->onReservedBuffers(IR, BufferIDs);
 282     return;
 283   }
 284
 285   for (HWEventListener *Listener : getListeners())
 286     Listener->onReleasedBuffers(IR, BufferIDs);
 287 }
 288
 289 } // namespace mca
 290 } // namespace llvm