llvm/lib/ProfileData/InstrProfCorrelator.cpp

   1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ProfileData/InstrProfCorrelator.h"
  10 #include "llvm/DebugInfo/DIContext.h"
  11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
  12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
  13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
  14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
  15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
  16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
  17 #include "llvm/Object/MachO.h"
  18 #include "llvm/Support/Debug.h"
  19 #include "llvm/Support/Format.h"
  20 #include "llvm/Support/WithColor.h"
  21 #include <optional>
  22
  23 #define DEBUG_TYPE "correlator"
  24
  25 using namespace llvm;
  26
  27 /// Get the __llvm_prf_cnts section.
  28 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
  29   for (auto &Section : Obj.sections())
  30     if (auto SectionName = Section.getName())
  31       if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
  32         return Section;
  33   return make_error<InstrProfError>(
  34       instrprof_error::unable_to_correlate_profile,
  35       "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
  36 }
  37
  38 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
  39 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
  40 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
  41
  42 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
  43 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
  44                                   const object::ObjectFile &Obj) {
  45   auto CountersSection = getCountersSection(Obj);
  46   if (auto Err = CountersSection.takeError())
  47     return std::move(Err);
  48   auto C = std::make_unique<Context>();
  49   C->Buffer = std::move(Buffer);
  50   C->CountersSectionStart = CountersSection->getAddress();
  51   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
  52   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
  53   return Expected<std::unique_ptr<Context>>(std::move(C));
  54 }
  55
  56 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  57 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
  58   auto DsymObjectsOrErr =
  59       object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
  60   if (auto Err = DsymObjectsOrErr.takeError())
  61     return std::move(Err);
  62   if (!DsymObjectsOrErr->empty()) {
  63     // TODO: Enable profile correlation when there are multiple objects in a
  64     // dSYM bundle.
  65     if (DsymObjectsOrErr->size() > 1)
  66       return make_error<InstrProfError>(
  67           instrprof_error::unable_to_correlate_profile,
  68           "using multiple objects is not yet supported");
  69     DebugInfoFilename = *DsymObjectsOrErr->begin();
  70   }
  71   auto BufferOrErr =
  72       errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
  73   if (auto Err = BufferOrErr.takeError())
  74     return std::move(Err);
  75
  76   return get(std::move(*BufferOrErr));
  77 }
  78
  79 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  80 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
  81   auto BinOrErr = object::createBinary(*Buffer);
  82   if (auto Err = BinOrErr.takeError())
  83     return std::move(Err);
  84
  85   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
  86     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
  87     if (auto Err = CtxOrErr.takeError())
  88       return std::move(Err);
  89     auto T = Obj->makeTriple();
  90     if (T.isArch64Bit())
  91       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
  92     if (T.isArch32Bit())
  93       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
  94   }
  95   return make_error<InstrProfError>(
  96       instrprof_error::unable_to_correlate_profile, "not an object file");
  97 }
  98
  99 std::optional<size_t> InstrProfCorrelator::getDataSize() const {
 100   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
 101     return C->getDataSize();
 102   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
 103     return C->getDataSize();
 104   }
 105   return {};
 106 }
 107
 108 namespace llvm {
 109
 110 template <>
 111 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
 112     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 113     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
 114                               std::move(Ctx)) {}
 115 template <>
 116 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
 117     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 118     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
 119                               std::move(Ctx)) {}
 120 template <>
 121 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
 122   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
 123 }
 124 template <>
 125 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
 126   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
 127 }
 128
 129 } // end namespace llvm
 130
 131 template <class IntPtrT>
 132 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
 133 InstrProfCorrelatorImpl<IntPtrT>::get(
 134     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
 135     const object::ObjectFile &Obj) {
 136   if (Obj.isELF() || Obj.isMachO()) {
 137     auto DICtx = DWARFContext::create(Obj);
 138     return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
 139                                                                std::move(Ctx));
 140   }
 141   return make_error<InstrProfError>(
 142       instrprof_error::unable_to_correlate_profile,
 143       "unsupported debug info format (only DWARF is supported)");
 144 }
 145
 146 template <class IntPtrT>
 147 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
 148   assert(Data.empty() && Names.empty() && NamesVec.empty());
 149   correlateProfileDataImpl(MaxWarnings);
 150   if (Data.empty() || NamesVec.empty())
 151     return make_error<InstrProfError>(
 152         instrprof_error::unable_to_correlate_profile,
 153         "could not find any profile metadata in debug info");
 154   auto Result =
 155       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
 156   CounterOffsets.clear();
 157   NamesVec.clear();
 158   return Result;
 159 }
 160
 161 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
 162   static void mapping(yaml::IO &io,
 163                       InstrProfCorrelator::CorrelationData &Data) {
 164     io.mapRequired("Probes", Data.Probes);
 165   }
 166 };
 167
 168 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
 169   static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
 170     io.mapRequired("Function Name", P.FunctionName);
 171     io.mapOptional("Linkage Name", P.LinkageName);
 172     io.mapRequired("CFG Hash", P.CFGHash);
 173     io.mapRequired("Counter Offset", P.CounterOffset);
 174     io.mapRequired("Num Counters", P.NumCounters);
 175     io.mapOptional("File", P.FilePath);
 176     io.mapOptional("Line", P.LineNumber);
 177   }
 178 };
 179
 180 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
 181   static const bool flow = false;
 182 };
 183
 184 template <class IntPtrT>
 185 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
 186                                                  raw_ostream &OS) {
 187   InstrProfCorrelator::CorrelationData Data;
 188   correlateProfileDataImpl(MaxWarnings, &Data);
 189   if (Data.Probes.empty())
 190     return make_error<InstrProfError>(
 191         instrprof_error::unable_to_correlate_profile,
 192         "could not find any profile metadata in debug info");
 193   yaml::Output YamlOS(OS);
 194   YamlOS << Data;
 195   return Error::success();
 196 }
 197
 198 template <class IntPtrT>
 199 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
 200                                                 uint64_t CFGHash,
 201                                                 IntPtrT CounterOffset,
 202                                                 IntPtrT FunctionPtr,
 203                                                 uint32_t NumCounters) {
 204   // Check if a probe was already added for this counter offset.
 205   if (!CounterOffsets.insert(CounterOffset).second)
 206     return;
 207   Data.push_back({
 208       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
 209       maybeSwap<uint64_t>(CFGHash),
 210       // In this mode, CounterPtr actually stores the section relative address
 211       // of the counter.
 212       maybeSwap<IntPtrT>(CounterOffset),
 213       maybeSwap<IntPtrT>(FunctionPtr),
 214       // TODO: Value profiling is not yet supported.
 215       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
 216       maybeSwap<uint32_t>(NumCounters),
 217       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
 218   });
 219   NamesVec.push_back(FunctionName.str());
 220 }
 221
 222 template <class IntPtrT>
 223 std::optional<uint64_t>
 224 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
 225   auto Locations = Die.getLocations(dwarf::DW_AT_location);
 226   if (!Locations) {
 227     consumeError(Locations.takeError());
 228     return {};
 229   }
 230   auto &DU = *Die.getDwarfUnit();
 231   auto AddressSize = DU.getAddressByteSize();
 232   for (auto &Location : *Locations) {
 233     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
 234     DWARFExpression Expr(Data, AddressSize);
 235     for (auto &Op : Expr) {
 236       if (Op.getCode() == dwarf::DW_OP_addr) {
 237         return Op.getRawOperand(0);
 238       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
 239         uint64_t Index = Op.getRawOperand(0);
 240         if (auto SA = DU.getAddrOffsetSectionItem(Index))
 241           return SA->Address;
 242       }
 243     }
 244   }
 245   return {};
 246 }
 247
 248 template <class IntPtrT>
 249 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
 250   const auto &ParentDie = Die.getParent();
 251   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
 252     return false;
 253   if (Die.getTag() != dwarf::DW_TAG_variable)
 254     return false;
 255   if (!ParentDie.isSubprogramDIE())
 256     return false;
 257   if (!Die.hasChildren())
 258     return false;
 259   if (const char *Name = Die.getName(DINameKind::ShortName))
 260     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
 261   return false;
 262 }
 263
 264 template <class IntPtrT>
 265 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
 266     int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
 267   bool UnlimitedWarnings = (MaxWarnings == 0);
 268   // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
 269   int NumSuppressedWarnings = -MaxWarnings;
 270   auto maybeAddProbe = [&](DWARFDie Die) {
 271     if (!isDIEOfProbe(Die))
 272       return;
 273     std::optional<const char *> FunctionName;
 274     std::optional<uint64_t> CFGHash;
 275     std::optional<uint64_t> CounterPtr = getLocation(Die);
 276     auto FnDie = Die.getParent();
 277     auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
 278     std::optional<uint64_t> NumCounters;
 279     for (const DWARFDie &Child : Die.children()) {
 280       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
 281         continue;
 282       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
 283       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
 284       if (!AnnotationFormName || !AnnotationFormValue)
 285         continue;
 286       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
 287       if (auto Err = AnnotationNameOrErr.takeError()) {
 288         consumeError(std::move(Err));
 289         continue;
 290       }
 291       StringRef AnnotationName = *AnnotationNameOrErr;
 292       if (AnnotationName.compare(
 293               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
 294         if (auto EC =
 295                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
 296           consumeError(std::move(EC));
 297       } else if (AnnotationName.compare(
 298                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
 299         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
 300       } else if (AnnotationName.compare(
 301                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
 302         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
 303       }
 304     }
 305     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
 306       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 307         WithColor::warning()
 308             << "Incomplete DIE for function " << FunctionName
 309             << ": CFGHash=" << CFGHash << "  CounterPtr=" << CounterPtr
 310             << "  NumCounters=" << NumCounters << "\n";
 311         LLVM_DEBUG(Die.dump(dbgs()));
 312       }
 313       return;
 314     }
 315     uint64_t CountersStart = this->Ctx->CountersSectionStart;
 316     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
 317     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
 318       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 319         WithColor::warning()
 320             << format("CounterPtr out of range for function %s: Actual=0x%x "
 321                       "Expected=[0x%x, 0x%x)\n",
 322                       *FunctionName, *CounterPtr, CountersStart, CountersEnd);
 323         LLVM_DEBUG(Die.dump(dbgs()));
 324       }
 325       return;
 326     }
 327     if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
 328       WithColor::warning() << format("Could not find address of function %s\n",
 329                                      *FunctionName);
 330       LLVM_DEBUG(Die.dump(dbgs()));
 331     }
 332     IntPtrT CounterOffset = *CounterPtr - CountersStart;
 333     if (Data) {
 334       InstrProfCorrelator::Probe P;
 335       P.FunctionName = *FunctionName;
 336       if (auto Name = FnDie.getName(DINameKind::LinkageName))
 337         P.LinkageName = Name;
 338       P.CFGHash = *CFGHash;
 339       P.CounterOffset = CounterOffset;
 340       P.NumCounters = *NumCounters;
 341       auto FilePath = FnDie.getDeclFile(
 342           DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
 343       if (!FilePath.empty())
 344         P.FilePath = FilePath;
 345       if (auto LineNumber = FnDie.getDeclLine())
 346         P.LineNumber = LineNumber;
 347       Data->Probes.push_back(P);
 348     } else {
 349       this->addProbe(*FunctionName, *CFGHash, CounterOffset,
 350                      FunctionPtr.value_or(0), *NumCounters);
 351     }
 352   };
 353   for (auto &CU : DICtx->normal_units())
 354     for (const auto &Entry : CU->dies())
 355       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 356   for (auto &CU : DICtx->dwo_units())
 357     for (const auto &Entry : CU->dies())
 358       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 359
 360   if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
 361     WithColor::warning() << format("Suppressed %d additional warnings\n",
 362                                    NumSuppressedWarnings);
 363 }