llvm/lib/ProfileData/InstrProfCorrelator.cpp

   1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ProfileData/InstrProfCorrelator.h"
  10 #include "llvm/DebugInfo/DIContext.h"
  11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
  12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
  13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
  14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
  15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
  16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
  17 #include "llvm/Object/MachO.h"
  18 #include "llvm/Support/Debug.h"
  19 #include <optional>
  20
  21 #define DEBUG_TYPE "correlator"
  22
  23 using namespace llvm;
  24
  25 /// Get the __llvm_prf_cnts section.
  26 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
  27   for (auto &Section : Obj.sections())
  28     if (auto SectionName = Section.getName())
  29       if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
  30         return Section;
  31   return make_error<InstrProfError>(
  32       instrprof_error::unable_to_correlate_profile,
  33       "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
  34 }
  35
  36 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
  37 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
  38 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
  39
  40 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
  41 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
  42                                   const object::ObjectFile &Obj) {
  43   auto CountersSection = getCountersSection(Obj);
  44   if (auto Err = CountersSection.takeError())
  45     return std::move(Err);
  46   auto C = std::make_unique<Context>();
  47   C->Buffer = std::move(Buffer);
  48   C->CountersSectionStart = CountersSection->getAddress();
  49   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
  50   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
  51   return Expected<std::unique_ptr<Context>>(std::move(C));
  52 }
  53
  54 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  55 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
  56   auto DsymObjectsOrErr =
  57       object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
  58   if (auto Err = DsymObjectsOrErr.takeError())
  59     return std::move(Err);
  60   if (!DsymObjectsOrErr->empty()) {
  61     // TODO: Enable profile correlation when there are multiple objects in a
  62     // dSYM bundle.
  63     if (DsymObjectsOrErr->size() > 1)
  64       return make_error<InstrProfError>(
  65           instrprof_error::unable_to_correlate_profile,
  66           "using multiple objects is not yet supported");
  67     DebugInfoFilename = *DsymObjectsOrErr->begin();
  68   }
  69   auto BufferOrErr =
  70       errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
  71   if (auto Err = BufferOrErr.takeError())
  72     return std::move(Err);
  73
  74   return get(std::move(*BufferOrErr));
  75 }
  76
  77 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  78 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
  79   auto BinOrErr = object::createBinary(*Buffer);
  80   if (auto Err = BinOrErr.takeError())
  81     return std::move(Err);
  82
  83   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
  84     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
  85     if (auto Err = CtxOrErr.takeError())
  86       return std::move(Err);
  87     auto T = Obj->makeTriple();
  88     if (T.isArch64Bit())
  89       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
  90     if (T.isArch32Bit())
  91       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
  92   }
  93   return make_error<InstrProfError>(
  94       instrprof_error::unable_to_correlate_profile, "not an object file");
  95 }
  96
  97 std::optional<size_t> InstrProfCorrelator::getDataSize() const {
  98   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
  99     return C->getDataSize();
 100   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
 101     return C->getDataSize();
 102   }
 103   return {};
 104 }
 105
 106 namespace llvm {
 107
 108 template <>
 109 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
 110     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 111     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
 112                               std::move(Ctx)) {}
 113 template <>
 114 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
 115     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 116     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
 117                               std::move(Ctx)) {}
 118 template <>
 119 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
 120   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
 121 }
 122 template <>
 123 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
 124   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
 125 }
 126
 127 } // end namespace llvm
 128
 129 template <class IntPtrT>
 130 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
 131 InstrProfCorrelatorImpl<IntPtrT>::get(
 132     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
 133     const object::ObjectFile &Obj) {
 134   if (Obj.isELF() || Obj.isMachO()) {
 135     auto DICtx = DWARFContext::create(Obj);
 136     return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
 137                                                                std::move(Ctx));
 138   }
 139   return make_error<InstrProfError>(
 140       instrprof_error::unable_to_correlate_profile,
 141       "unsupported debug info format (only DWARF is supported)");
 142 }
 143
 144 template <class IntPtrT>
 145 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
 146   assert(Data.empty() && Names.empty() && NamesVec.empty());
 147   correlateProfileDataImpl();
 148   if (Data.empty() || NamesVec.empty())
 149     return make_error<InstrProfError>(
 150         instrprof_error::unable_to_correlate_profile,
 151         "could not find any profile metadata in debug info");
 152   auto Result =
 153       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
 154   CounterOffsets.clear();
 155   NamesVec.clear();
 156   return Result;
 157 }
 158
 159 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
 160   static void mapping(yaml::IO &io,
 161                       InstrProfCorrelator::CorrelationData &Data) {
 162     io.mapRequired("Probes", Data.Probes);
 163   }
 164 };
 165
 166 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
 167   static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
 168     io.mapRequired("Function Name", P.FunctionName);
 169     io.mapOptional("Linkage Name", P.LinkageName);
 170     io.mapRequired("CFG Hash", P.CFGHash);
 171     io.mapRequired("Counter Offset", P.CounterOffset);
 172     io.mapRequired("Num Counters", P.NumCounters);
 173     io.mapOptional("File", P.FilePath);
 174     io.mapOptional("Line", P.LineNumber);
 175   }
 176 };
 177
 178 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
 179   static const bool flow = false;
 180 };
 181
 182 template <class IntPtrT>
 183 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(raw_ostream &OS) {
 184   InstrProfCorrelator::CorrelationData Data;
 185   correlateProfileDataImpl(&Data);
 186   if (Data.Probes.empty())
 187     return make_error<InstrProfError>(
 188         instrprof_error::unable_to_correlate_profile,
 189         "could not find any profile metadata in debug info");
 190   yaml::Output YamlOS(OS);
 191   YamlOS << Data;
 192   return Error::success();
 193 }
 194
 195 template <class IntPtrT>
 196 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
 197                                                 uint64_t CFGHash,
 198                                                 IntPtrT CounterOffset,
 199                                                 IntPtrT FunctionPtr,
 200                                                 uint32_t NumCounters) {
 201   // Check if a probe was already added for this counter offset.
 202   if (!CounterOffsets.insert(CounterOffset).second)
 203     return;
 204   Data.push_back({
 205       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
 206       maybeSwap<uint64_t>(CFGHash),
 207       // In this mode, CounterPtr actually stores the section relative address
 208       // of the counter.
 209       maybeSwap<IntPtrT>(CounterOffset),
 210       maybeSwap<IntPtrT>(FunctionPtr),
 211       // TODO: Value profiling is not yet supported.
 212       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
 213       maybeSwap<uint32_t>(NumCounters),
 214       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
 215   });
 216   NamesVec.push_back(FunctionName.str());
 217 }
 218
 219 template <class IntPtrT>
 220 std::optional<uint64_t>
 221 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
 222   auto Locations = Die.getLocations(dwarf::DW_AT_location);
 223   if (!Locations) {
 224     consumeError(Locations.takeError());
 225     return {};
 226   }
 227   auto &DU = *Die.getDwarfUnit();
 228   auto AddressSize = DU.getAddressByteSize();
 229   for (auto &Location : *Locations) {
 230     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
 231     DWARFExpression Expr(Data, AddressSize);
 232     for (auto &Op : Expr) {
 233       if (Op.getCode() == dwarf::DW_OP_addr) {
 234         return Op.getRawOperand(0);
 235       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
 236         uint64_t Index = Op.getRawOperand(0);
 237         if (auto SA = DU.getAddrOffsetSectionItem(Index))
 238           return SA->Address;
 239       }
 240     }
 241   }
 242   return {};
 243 }
 244
 245 template <class IntPtrT>
 246 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
 247   const auto &ParentDie = Die.getParent();
 248   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
 249     return false;
 250   if (Die.getTag() != dwarf::DW_TAG_variable)
 251     return false;
 252   if (!ParentDie.isSubprogramDIE())
 253     return false;
 254   if (!Die.hasChildren())
 255     return false;
 256   if (const char *Name = Die.getName(DINameKind::ShortName))
 257     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
 258   return false;
 259 }
 260
 261 template <class IntPtrT>
 262 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
 263     InstrProfCorrelator::CorrelationData *Data) {
 264   auto maybeAddProbe = [&](DWARFDie Die) {
 265     if (!isDIEOfProbe(Die))
 266       return;
 267     std::optional<const char *> FunctionName;
 268     std::optional<uint64_t> CFGHash;
 269     std::optional<uint64_t> CounterPtr = getLocation(Die);
 270     auto FnDie = Die.getParent();
 271     auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
 272     std::optional<uint64_t> NumCounters;
 273     for (const DWARFDie &Child : Die.children()) {
 274       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
 275         continue;
 276       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
 277       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
 278       if (!AnnotationFormName || !AnnotationFormValue)
 279         continue;
 280       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
 281       if (auto Err = AnnotationNameOrErr.takeError()) {
 282         consumeError(std::move(Err));
 283         continue;
 284       }
 285       StringRef AnnotationName = *AnnotationNameOrErr;
 286       if (AnnotationName.compare(
 287               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
 288         if (auto EC =
 289                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
 290           consumeError(std::move(EC));
 291       } else if (AnnotationName.compare(
 292                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
 293         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
 294       } else if (AnnotationName.compare(
 295                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
 296         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
 297       }
 298     }
 299     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
 300       LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
 301                         << FunctionName << "\n\tCFGHash: " << CFGHash
 302                         << "\n\tCounterPtr: " << CounterPtr
 303                         << "\n\tNumCounters: " << NumCounters);
 304       LLVM_DEBUG(Die.dump(dbgs()));
 305       return;
 306     }
 307     uint64_t CountersStart = this->Ctx->CountersSectionStart;
 308     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
 309     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
 310       LLVM_DEBUG(
 311           dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
 312                  << FunctionName << "\n\tExpected: [0x"
 313                  << Twine::utohexstr(CountersStart) << ", 0x"
 314                  << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
 315                  << Twine::utohexstr(*CounterPtr));
 316       LLVM_DEBUG(Die.dump(dbgs()));
 317       return;
 318     }
 319     if (!FunctionPtr) {
 320       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
 321                         << "\n");
 322       LLVM_DEBUG(Die.dump(dbgs()));
 323     }
 324     IntPtrT CounterOffset = *CounterPtr - CountersStart;
 325     if (Data) {
 326       InstrProfCorrelator::Probe P;
 327       P.FunctionName = *FunctionName;
 328       if (auto Name = FnDie.getName(DINameKind::LinkageName))
 329         P.LinkageName = Name;
 330       P.CFGHash = *CFGHash;
 331       P.CounterOffset = CounterOffset;
 332       P.NumCounters = *NumCounters;
 333       auto FilePath = FnDie.getDeclFile(
 334           DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
 335       if (!FilePath.empty())
 336         P.FilePath = FilePath;
 337       if (auto LineNumber = FnDie.getDeclLine())
 338         P.LineNumber = LineNumber;
 339       Data->Probes.push_back(P);
 340     } else {
 341       this->addProbe(*FunctionName, *CFGHash, CounterOffset,
 342                      FunctionPtr.value_or(0), *NumCounters);
 343     }
 344   };
 345   for (auto &CU : DICtx->normal_units())
 346     for (const auto &Entry : CU->dies())
 347       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 348   for (auto &CU : DICtx->dwo_units())
 349     for (const auto &Entry : CU->dies())
 350       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 351 }