llvm/lib/ProfileData/InstrProfCorrelator.cpp

   1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ProfileData/InstrProfCorrelator.h"
  10 #include "llvm/DebugInfo/DIContext.h"
  11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
  12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
  13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
  14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
  15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
  16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
  17 #include "llvm/Object/MachO.h"
  18 #include "llvm/Support/Debug.h"
  19 #include "llvm/Support/Format.h"
  20 #include "llvm/Support/WithColor.h"
  21 #include <optional>
  22
  23 #define DEBUG_TYPE "correlator"
  24
  25 using namespace llvm;
  26
  27 /// Get profile section.
  28 Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj,
  29                                                  InstrProfSectKind IPSK) {
  30   Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
  31   std::string ExpectedSectionName =
  32       getInstrProfSectionName(IPSK, ObjFormat,
  33                               /*AddSegmentInfo=*/false);
  34   for (auto &Section : Obj.sections())
  35     if (auto SectionName = Section.getName())
  36       if (SectionName.get() == ExpectedSectionName)
  37         return Section;
  38   return make_error<InstrProfError>(
  39       instrprof_error::unable_to_correlate_profile,
  40       "could not find section (" + Twine(ExpectedSectionName) + ")");
  41 }
  42
  43 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
  44 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
  45 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
  46
  47 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
  48 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
  49                                   const object::ObjectFile &Obj) {
  50   auto CountersSection = getInstrProfSection(Obj, IPSK_cnts);
  51   if (auto Err = CountersSection.takeError())
  52     return std::move(Err);
  53   auto C = std::make_unique<Context>();
  54   C->Buffer = std::move(Buffer);
  55   C->CountersSectionStart = CountersSection->getAddress();
  56   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
  57   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
  58   return Expected<std::unique_ptr<Context>>(std::move(C));
  59 }
  60
  61 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  62 InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) {
  63   if (FileKind == DEBUG_INFO) {
  64     auto DsymObjectsOrErr =
  65         object::MachOObjectFile::findDsymObjectMembers(Filename);
  66     if (auto Err = DsymObjectsOrErr.takeError())
  67       return std::move(Err);
  68     if (!DsymObjectsOrErr->empty()) {
  69       // TODO: Enable profile correlation when there are multiple objects in a
  70       // dSYM bundle.
  71       if (DsymObjectsOrErr->size() > 1)
  72         return make_error<InstrProfError>(
  73             instrprof_error::unable_to_correlate_profile,
  74             "using multiple objects is not yet supported");
  75       Filename = *DsymObjectsOrErr->begin();
  76     }
  77   }
  78   auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));
  79   if (auto Err = BufferOrErr.takeError())
  80     return std::move(Err);
  81
  82   return get(std::move(*BufferOrErr), FileKind);
  83 }
  84
  85 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  86 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,
  87                          ProfCorrelatorKind FileKind) {
  88   auto BinOrErr = object::createBinary(*Buffer);
  89   if (auto Err = BinOrErr.takeError())
  90     return std::move(Err);
  91
  92   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
  93     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
  94     if (auto Err = CtxOrErr.takeError())
  95       return std::move(Err);
  96     auto T = Obj->makeTriple();
  97     if (T.isArch64Bit())
  98       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj,
  99                                                     FileKind);
 100     if (T.isArch32Bit())
 101       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj,
 102                                                     FileKind);
 103   }
 104   return make_error<InstrProfError>(
 105       instrprof_error::unable_to_correlate_profile, "not an object file");
 106 }
 107
 108 std::optional<size_t> InstrProfCorrelator::getDataSize() const {
 109   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
 110     return C->getDataSize();
 111   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
 112     return C->getDataSize();
 113   }
 114   return {};
 115 }
 116
 117 namespace llvm {
 118
 119 template <>
 120 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
 121     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 122     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
 123                               std::move(Ctx)) {}
 124 template <>
 125 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
 126     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 127     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
 128                               std::move(Ctx)) {}
 129 template <>
 130 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
 131   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
 132 }
 133 template <>
 134 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
 135   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
 136 }
 137
 138 } // end namespace llvm
 139
 140 template <class IntPtrT>
 141 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
 142 InstrProfCorrelatorImpl<IntPtrT>::get(
 143     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
 144     const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) {
 145   if (FileKind == DEBUG_INFO) {
 146     if (Obj.isELF() || Obj.isMachO()) {
 147       auto DICtx = DWARFContext::create(Obj);
 148       return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(
 149           std::move(DICtx), std::move(Ctx));
 150     }
 151     return make_error<InstrProfError>(
 152         instrprof_error::unable_to_correlate_profile,
 153         "unsupported debug info format (only DWARF is supported)");
 154   }
 155   return make_error<InstrProfError>(
 156       instrprof_error::unable_to_correlate_profile,
 157       "unsupported correlation file type (only DWARF is supported)");
 158 }
 159
 160 template <class IntPtrT>
 161 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
 162   assert(Data.empty() && Names.empty() && NamesVec.empty());
 163   correlateProfileDataImpl(MaxWarnings);
 164   if (this->Data.empty())
 165     return make_error<InstrProfError>(
 166         instrprof_error::unable_to_correlate_profile,
 167         "could not find any profile data metadata in correlated file");
 168   Error Result = correlateProfileNameImpl();
 169   this->CounterOffsets.clear();
 170   this->NamesVec.clear();
 171   return Result;
 172 }
 173
 174 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
 175   static void mapping(yaml::IO &io,
 176                       InstrProfCorrelator::CorrelationData &Data) {
 177     io.mapRequired("Probes", Data.Probes);
 178   }
 179 };
 180
 181 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
 182   static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
 183     io.mapRequired("Function Name", P.FunctionName);
 184     io.mapOptional("Linkage Name", P.LinkageName);
 185     io.mapRequired("CFG Hash", P.CFGHash);
 186     io.mapRequired("Counter Offset", P.CounterOffset);
 187     io.mapRequired("Num Counters", P.NumCounters);
 188     io.mapOptional("File", P.FilePath);
 189     io.mapOptional("Line", P.LineNumber);
 190   }
 191 };
 192
 193 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
 194   static const bool flow = false;
 195 };
 196
 197 template <class IntPtrT>
 198 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
 199                                                  raw_ostream &OS) {
 200   InstrProfCorrelator::CorrelationData Data;
 201   correlateProfileDataImpl(MaxWarnings, &Data);
 202   if (Data.Probes.empty())
 203     return make_error<InstrProfError>(
 204         instrprof_error::unable_to_correlate_profile,
 205         "could not find any profile data metadata in debug info");
 206   yaml::Output YamlOS(OS);
 207   YamlOS << Data;
 208   return Error::success();
 209 }
 210
 211 template <class IntPtrT>
 212 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
 213                                                 uint64_t CFGHash,
 214                                                 IntPtrT CounterOffset,
 215                                                 IntPtrT FunctionPtr,
 216                                                 uint32_t NumCounters) {
 217   // Check if a probe was already added for this counter offset.
 218   if (!CounterOffsets.insert(CounterOffset).second)
 219     return;
 220   Data.push_back({
 221       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
 222       maybeSwap<uint64_t>(CFGHash),
 223       // In this mode, CounterPtr actually stores the section relative address
 224       // of the counter.
 225       maybeSwap<IntPtrT>(CounterOffset),
 226       // TODO: MC/DC is not yet supported.
 227       /*BitmapOffset=*/maybeSwap<IntPtrT>(0),
 228       maybeSwap<IntPtrT>(FunctionPtr),
 229       // TODO: Value profiling is not yet supported.
 230       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
 231       maybeSwap<uint32_t>(NumCounters),
 232       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
 233       // TODO: MC/DC is not yet supported.
 234       /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
 235   });
 236   NamesVec.push_back(FunctionName.str());
 237 }
 238
 239 template <class IntPtrT>
 240 std::optional<uint64_t>
 241 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
 242   auto Locations = Die.getLocations(dwarf::DW_AT_location);
 243   if (!Locations) {
 244     consumeError(Locations.takeError());
 245     return {};
 246   }
 247   auto &DU = *Die.getDwarfUnit();
 248   auto AddressSize = DU.getAddressByteSize();
 249   for (auto &Location : *Locations) {
 250     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
 251     DWARFExpression Expr(Data, AddressSize);
 252     for (auto &Op : Expr) {
 253       if (Op.getCode() == dwarf::DW_OP_addr) {
 254         return Op.getRawOperand(0);
 255       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
 256         uint64_t Index = Op.getRawOperand(0);
 257         if (auto SA = DU.getAddrOffsetSectionItem(Index))
 258           return SA->Address;
 259       }
 260     }
 261   }
 262   return {};
 263 }
 264
 265 template <class IntPtrT>
 266 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
 267   const auto &ParentDie = Die.getParent();
 268   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
 269     return false;
 270   if (Die.getTag() != dwarf::DW_TAG_variable)
 271     return false;
 272   if (!ParentDie.isSubprogramDIE())
 273     return false;
 274   if (!Die.hasChildren())
 275     return false;
 276   if (const char *Name = Die.getName(DINameKind::ShortName))
 277     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
 278   return false;
 279 }
 280
 281 template <class IntPtrT>
 282 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
 283     int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
 284   bool UnlimitedWarnings = (MaxWarnings == 0);
 285   // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
 286   int NumSuppressedWarnings = -MaxWarnings;
 287   auto maybeAddProbe = [&](DWARFDie Die) {
 288     if (!isDIEOfProbe(Die))
 289       return;
 290     std::optional<const char *> FunctionName;
 291     std::optional<uint64_t> CFGHash;
 292     std::optional<uint64_t> CounterPtr = getLocation(Die);
 293     auto FnDie = Die.getParent();
 294     auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
 295     std::optional<uint64_t> NumCounters;
 296     for (const DWARFDie &Child : Die.children()) {
 297       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
 298         continue;
 299       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
 300       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
 301       if (!AnnotationFormName || !AnnotationFormValue)
 302         continue;
 303       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
 304       if (auto Err = AnnotationNameOrErr.takeError()) {
 305         consumeError(std::move(Err));
 306         continue;
 307       }
 308       StringRef AnnotationName = *AnnotationNameOrErr;
 309       if (AnnotationName.compare(
 310               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
 311         if (auto EC =
 312                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
 313           consumeError(std::move(EC));
 314       } else if (AnnotationName.compare(
 315                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
 316         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
 317       } else if (AnnotationName.compare(
 318                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
 319         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
 320       }
 321     }
 322     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
 323       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 324         WithColor::warning()
 325             << "Incomplete DIE for function " << FunctionName
 326             << ": CFGHash=" << CFGHash << "  CounterPtr=" << CounterPtr
 327             << "  NumCounters=" << NumCounters << "\n";
 328         LLVM_DEBUG(Die.dump(dbgs()));
 329       }
 330       return;
 331     }
 332     uint64_t CountersStart = this->Ctx->CountersSectionStart;
 333     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
 334     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
 335       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 336         WithColor::warning()
 337             << format("CounterPtr out of range for function %s: Actual=0x%x "
 338                       "Expected=[0x%x, 0x%x)\n",
 339                       *FunctionName, *CounterPtr, CountersStart, CountersEnd);
 340         LLVM_DEBUG(Die.dump(dbgs()));
 341       }
 342       return;
 343     }
 344     if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
 345       WithColor::warning() << format("Could not find address of function %s\n",
 346                                      *FunctionName);
 347       LLVM_DEBUG(Die.dump(dbgs()));
 348     }
 349     IntPtrT CounterOffset = *CounterPtr - CountersStart;
 350     if (Data) {
 351       InstrProfCorrelator::Probe P;
 352       P.FunctionName = *FunctionName;
 353       if (auto Name = FnDie.getName(DINameKind::LinkageName))
 354         P.LinkageName = Name;
 355       P.CFGHash = *CFGHash;
 356       P.CounterOffset = CounterOffset;
 357       P.NumCounters = *NumCounters;
 358       auto FilePath = FnDie.getDeclFile(
 359           DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
 360       if (!FilePath.empty())
 361         P.FilePath = FilePath;
 362       if (auto LineNumber = FnDie.getDeclLine())
 363         P.LineNumber = LineNumber;
 364       Data->Probes.push_back(P);
 365     } else {
 366       this->addProbe(*FunctionName, *CFGHash, CounterOffset,
 367                      FunctionPtr.value_or(0), *NumCounters);
 368     }
 369   };
 370   for (auto &CU : DICtx->normal_units())
 371     for (const auto &Entry : CU->dies())
 372       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 373   for (auto &CU : DICtx->dwo_units())
 374     for (const auto &Entry : CU->dies())
 375       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 376
 377   if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
 378     WithColor::warning() << format("Suppressed %d additional warnings\n",
 379                                    NumSuppressedWarnings);
 380 }
 381
 382 template <class IntPtrT>
 383 Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
 384   if (this->NamesVec.empty()) {
 385     return make_error<InstrProfError>(
 386         instrprof_error::unable_to_correlate_profile,
 387         "could not find any profile name metadata in debug info");
 388   }
 389   auto Result =
 390       collectGlobalObjectNameStrings(this->NamesVec,
 391                                      /*doCompression=*/false, this->Names);
 392   return Result;
 393 }