llvm/lib/Target/X86/X86DiscriminateMemOps.cpp

   1 //===- X86DiscriminateMemOps.cpp - Unique IDs for Mem Ops -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// This pass aids profile-driven cache prefetch insertion by ensuring all
  10 /// instructions that have a memory operand are distinguishible from each other.
  11 ///
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "X86.h"
  15 #include "X86InstrBuilder.h"
  16 #include "X86InstrInfo.h"
  17 #include "X86MachineFunctionInfo.h"
  18 #include "X86Subtarget.h"
  19 #include "llvm/CodeGen/MachineFunctionPass.h"
  20 #include "llvm/CodeGen/MachineModuleInfo.h"
  21 #include "llvm/IR/DebugInfoMetadata.h"
  22 #include "llvm/ProfileData/SampleProf.h"
  23 #include "llvm/ProfileData/SampleProfReader.h"
  24 #include "llvm/Support/Debug.h"
  25 #include "llvm/Transforms/IPO/SampleProfile.h"
  26 #include <optional>
  27 using namespace llvm;
  28
  29 #define DEBUG_TYPE "x86-discriminate-memops"
  30
  31 static cl::opt<bool> EnableDiscriminateMemops(
  32     DEBUG_TYPE, cl::init(false),
  33     cl::desc("Generate unique debug info for each instruction with a memory "
  34              "operand. Should be enabled for profile-driven cache prefetching, "
  35              "both in the build of the binary being profiled, as well as in "
  36              "the build of the binary consuming the profile."),
  37     cl::Hidden);
  38
  39 static cl::opt<bool> BypassPrefetchInstructions(
  40     "x86-bypass-prefetch-instructions", cl::init(true),
  41     cl::desc("When discriminating instructions with memory operands, ignore "
  42              "prefetch instructions. This ensures the other memory operand "
  43              "instructions have the same identifiers after inserting "
  44              "prefetches, allowing for successive insertions."),
  45     cl::Hidden);
  46
  47 namespace {
  48
  49 using Location = std::pair<StringRef, unsigned>;
  50
  51 Location diToLocation(const DILocation *Loc) {
  52   return std::make_pair(Loc->getFilename(), Loc->getLine());
  53 }
  54
  55 /// Ensure each instruction having a memory operand has a distinct <LineNumber,
  56 /// Discriminator> pair.
  57 void updateDebugInfo(MachineInstr *MI, const DILocation *Loc) {
  58   DebugLoc DL(Loc);
  59   MI->setDebugLoc(DL);
  60 }
  61
  62 class X86DiscriminateMemOps : public MachineFunctionPass {
  63   bool runOnMachineFunction(MachineFunction &MF) override;
  64   StringRef getPassName() const override {
  65     return "X86 Discriminate Memory Operands";
  66   }
  67
  68 public:
  69   static char ID;
  70
  71   /// Default construct and initialize the pass.
  72   X86DiscriminateMemOps();
  73 };
  74
  75 bool IsPrefetchOpcode(unsigned Opcode) {
  76   return Opcode == X86::PREFETCHNTA || Opcode == X86::PREFETCHT0 ||
  77          Opcode == X86::PREFETCHT1 || Opcode == X86::PREFETCHT2 ||
  78          Opcode == X86::PREFETCHIT0 || Opcode == X86::PREFETCHIT1;
  79 }
  80 } // end anonymous namespace
  81
  82 //===----------------------------------------------------------------------===//
  83 //            Implementation
  84 //===----------------------------------------------------------------------===//
  85
  86 char X86DiscriminateMemOps::ID = 0;
  87
  88 /// Default construct and initialize the pass.
  89 X86DiscriminateMemOps::X86DiscriminateMemOps() : MachineFunctionPass(ID) {}
  90
  91 bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) {
  92   if (!EnableDiscriminateMemops)
  93     return false;
  94
  95   DISubprogram *FDI = MF.getFunction().getSubprogram();
  96   if (!FDI || !FDI->getUnit()->getDebugInfoForProfiling())
  97     return false;
  98
  99   // Have a default DILocation, if we find instructions with memops that don't
 100   // have any debug info.
 101   const DILocation *ReferenceDI =
 102       DILocation::get(FDI->getContext(), FDI->getLine(), 0, FDI);
 103   assert(ReferenceDI && "ReferenceDI should not be nullptr");
 104   DenseMap<Location, unsigned> MemOpDiscriminators;
 105   MemOpDiscriminators[diToLocation(ReferenceDI)] = 0;
 106
 107   // Figure out the largest discriminator issued for each Location. When we
 108   // issue new discriminators, we can thus avoid issuing discriminators
 109   // belonging to instructions that don't have memops. This isn't a requirement
 110   // for the goals of this pass, however, it avoids unnecessary ambiguity.
 111   for (auto &MBB : MF) {
 112     for (auto &MI : MBB) {
 113       const auto &DI = MI.getDebugLoc();
 114       if (!DI)
 115         continue;
 116       if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
 117         continue;
 118       Location Loc = diToLocation(DI);
 119       MemOpDiscriminators[Loc] =
 120           std::max(MemOpDiscriminators[Loc], DI->getBaseDiscriminator());
 121     }
 122   }
 123
 124   // Keep track of the discriminators seen at each Location. If an instruction's
 125   // DebugInfo has a Location and discriminator we've already seen, replace its
 126   // discriminator with a new one, to guarantee uniqueness.
 127   DenseMap<Location, DenseSet<unsigned>> Seen;
 128
 129   bool Changed = false;
 130   for (auto &MBB : MF) {
 131     for (auto &MI : MBB) {
 132       if (X86II::getMemoryOperandNo(MI.getDesc().TSFlags) < 0)
 133         continue;
 134       if (BypassPrefetchInstructions && IsPrefetchOpcode(MI.getDesc().Opcode))
 135         continue;
 136       const DILocation *DI = MI.getDebugLoc();
 137       bool HasDebug = DI;
 138       if (!HasDebug) {
 139         DI = ReferenceDI;
 140       }
 141       Location L = diToLocation(DI);
 142       DenseSet<unsigned> &Set = Seen[L];
 143       const std::pair<DenseSet<unsigned>::iterator, bool> TryInsert =
 144           Set.insert(DI->getBaseDiscriminator());
 145       if (!TryInsert.second || !HasDebug) {
 146         unsigned BF, DF, CI = 0;
 147         DILocation::decodeDiscriminator(DI->getDiscriminator(), BF, DF, CI);
 148         std::optional<unsigned> EncodedDiscriminator =
 149             DILocation::encodeDiscriminator(MemOpDiscriminators[L] + 1, DF, CI);
 150
 151         if (!EncodedDiscriminator) {
 152           // FIXME(mtrofin): The assumption is that this scenario is infrequent/OK
 153           // not to support. If evidence points otherwise, we can explore synthesizeing
 154           // unique DIs by adding fake line numbers, or by constructing 64 bit
 155           // discriminators.
 156           LLVM_DEBUG(dbgs() << "Unable to create a unique discriminator "
 157                      "for instruction with memory operand in: "
 158                      << DI->getFilename() << " Line: " << DI->getLine()
 159                      << " Column: " << DI->getColumn()
 160                      << ". This is likely due to a large macro expansion. \n");
 161           continue;
 162         }
 163         // Since we were able to encode, bump the MemOpDiscriminators.
 164         ++MemOpDiscriminators[L];
 165         DI = DI->cloneWithDiscriminator(*EncodedDiscriminator);
 166         assert(DI && "DI should not be nullptr");
 167         updateDebugInfo(&MI, DI);
 168         Changed = true;
 169         std::pair<DenseSet<unsigned>::iterator, bool> MustInsert =
 170             Set.insert(DI->getBaseDiscriminator());
 171         (void)MustInsert; // Silence warning in release build.
 172         assert(MustInsert.second && "New discriminator shouldn't be present in set");
 173       }
 174
 175       // Bump the reference DI to avoid cramming discriminators on line 0.
 176       // FIXME(mtrofin): pin ReferenceDI on blocks or first instruction with DI
 177       // in a block. It's more consistent than just relying on the last memop
 178       // instruction we happened to see.
 179       ReferenceDI = DI;
 180     }
 181   }
 182   return Changed;
 183 }
 184
 185 FunctionPass *llvm::createX86DiscriminateMemOpsPass() {
 186   return new X86DiscriminateMemOps();
 187 }