llvm/lib/CodeGen/PseudoProbeInserter.cpp

   1 //===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements PseudoProbeInserter pass, which inserts pseudo probe
  10 // annotations for call instructions with a pseudo-probe-specific dwarf
  11 // discriminator. such discriminator indicates that the call instruction comes
  12 // with a pseudo probe, and the discriminator value holds information to
  13 // identify the corresponding counter.
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "llvm/CodeGen/MachineBasicBlock.h"
  17 #include "llvm/CodeGen/MachineFunctionPass.h"
  18 #include "llvm/CodeGen/MachineInstr.h"
  19 #include "llvm/CodeGen/TargetInstrInfo.h"
  20 #include "llvm/IR/DebugInfoMetadata.h"
  21 #include "llvm/IR/PseudoProbe.h"
  22 #include "llvm/InitializePasses.h"
  23 #include "llvm/MC/MCPseudoProbe.h"
  24 #include "llvm/Target/TargetMachine.h"
  25 #include <unordered_set>
  26
  27 #define DEBUG_TYPE "pseudo-probe-inserter"
  28
  29 using namespace llvm;
  30
  31 namespace {
  32 class PseudoProbeInserter : public MachineFunctionPass {
  33 public:
  34   static char ID;
  35
  36   PseudoProbeInserter() : MachineFunctionPass(ID) {
  37     initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
  38   }
  39
  40   StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
  41
  42   void getAnalysisUsage(AnalysisUsage &AU) const override {
  43     AU.setPreservesAll();
  44     MachineFunctionPass::getAnalysisUsage(AU);
  45   }
  46
  47   bool doInitialization(Module &M) override {
  48     ShouldRun = M.getNamedMetadata(PseudoProbeDescMetadataName);
  49     return false;
  50   }
  51
  52   bool runOnMachineFunction(MachineFunction &MF) override {
  53     if (!ShouldRun)
  54       return false;
  55     const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
  56     bool Changed = false;
  57     for (MachineBasicBlock &MBB : MF) {
  58       MachineInstr *FirstInstr = nullptr;
  59       for (MachineInstr &MI : MBB) {
  60         if (!MI.isPseudo())
  61           FirstInstr = &MI;
  62         if (MI.isCall()) {
  63           if (DILocation *DL = MI.getDebugLoc()) {
  64             auto Value = DL->getDiscriminator();
  65             if (DILocation::isPseudoProbeDiscriminator(Value)) {
  66               BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
  67                   .addImm(getFuncGUID(MF.getFunction().getParent(), DL))
  68                   .addImm(
  69                       PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
  70                   .addImm(
  71                       PseudoProbeDwarfDiscriminator::extractProbeType(Value))
  72                   .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
  73                       Value));
  74               Changed = true;
  75             }
  76           }
  77         }
  78       }
  79
  80       // Walk the block backwards, move PSEUDO_PROBE before the first real
  81       // instruction to fix out-of-order probes. There is a problem with probes
  82       // as the terminator of the block. During the offline counts processing,
  83       // the samples collected on the first physical instruction following a
  84       // probe will be counted towards the probe. This logically equals to
  85       // treating the instruction next to a probe as if it is from the same
  86       // block of the probe. This is accurate most of the time unless the
  87       // instruction can be reached from multiple flows, which means it actually
  88       // starts a new block. Samples collected on such probes may cause
  89       // imprecision with the counts inference algorithm. Fortunately, if
  90       // there are still other native instructions preceding the probe we can
  91       // use them as a place holder to collect samples for the probe.
  92       if (FirstInstr) {
  93         auto MII = MBB.rbegin();
  94         while (MII != MBB.rend()) {
  95           // Skip all pseudo probes followed by a real instruction since they
  96           // are not dangling.
  97           if (!MII->isPseudo())
  98             break;
  99           auto Cur = MII++;
 100           if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)
 101             continue;
 102           // Move the dangling probe before FirstInstr.
 103           auto *ProbeInstr = &*Cur;
 104           MBB.remove(ProbeInstr);
 105           MBB.insert(FirstInstr, ProbeInstr);
 106           Changed = true;
 107         }
 108       } else {
 109         // Probes not surrounded by any real instructions in the same block are
 110         // called dangling probes. Since there's no good way to pick up a sample
 111         // collection point for dangling probes at compile time, they are being
 112         // removed so that the profile correlation tool will not report any
 113         // samples collected for them and it's up to the counts inference tool
 114         // to get them a reasonable count.
 115         SmallVector<MachineInstr *, 4> ToBeRemoved;
 116         for (MachineInstr &MI : MBB) {
 117           if (MI.isPseudoProbe())
 118             ToBeRemoved.push_back(&MI);
 119         }
 120
 121         for (auto *MI : ToBeRemoved)
 122           MI->eraseFromParent();
 123
 124         Changed |= !ToBeRemoved.empty();
 125       }
 126     }
 127
 128     return Changed;
 129   }
 130
 131 private:
 132   uint64_t getFuncGUID(Module *M, DILocation *DL) {
 133     auto *SP = DL->getScope()->getSubprogram();
 134     auto Name = SP->getLinkageName();
 135     if (Name.empty())
 136       Name = SP->getName();
 137     return Function::getGUID(Name);
 138   }
 139
 140   bool ShouldRun = false;
 141 };
 142 } // namespace
 143
 144 char PseudoProbeInserter::ID = 0;
 145 INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
 146                       "Insert pseudo probe annotations for value profiling",
 147                       false, false)
 148 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
 149 INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
 150                     "Insert pseudo probe annotations for value profiling",
 151                     false, false)
 152
 153 FunctionPass *llvm::createPseudoProbeInserter() {
 154   return new PseudoProbeInserter();
 155 }