llvm/lib/ProfileData/InstrProfCorrelator.cpp

   1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ProfileData/InstrProfCorrelator.h"
  10 #include "llvm/DebugInfo/DIContext.h"
  11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
  12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
  13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
  14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
  15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
  16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
  17 #include "llvm/Object/MachO.h"
  18 #include "llvm/Support/Debug.h"
  19 #include "llvm/Support/Format.h"
  20 #include "llvm/Support/WithColor.h"
  21 #include <optional>
  22
  23 #define DEBUG_TYPE "correlator"
  24
  25 using namespace llvm;
  26
  27 /// Get profile section.
  28 Expected<object::SectionRef> getInstrProfSection(const object::ObjectFile &Obj,
  29                                                  InstrProfSectKind IPSK) {
  30   // On COFF, the getInstrProfSectionName returns the section names may followed
  31   // by "$M". The linker removes the dollar and everything after it in the final
  32   // binary. Do the same to match.
  33   Triple::ObjectFormatType ObjFormat = Obj.getTripleObjectFormat();
  34   auto StripSuffix = [ObjFormat](StringRef N) {
  35     return ObjFormat == Triple::COFF ? N.split('$').first : N;
  36   };
  37   std::string ExpectedSectionName =
  38       getInstrProfSectionName(IPSK, ObjFormat,
  39                               /*AddSegmentInfo=*/false);
  40   ExpectedSectionName = StripSuffix(ExpectedSectionName);
  41   for (auto &Section : Obj.sections()) {
  42     if (auto SectionName = Section.getName())
  43       if (*SectionName == ExpectedSectionName)
  44         return Section;
  45   }
  46   return make_error<InstrProfError>(
  47       instrprof_error::unable_to_correlate_profile,
  48       "could not find section (" + Twine(ExpectedSectionName) + ")");
  49 }
  50
  51 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
  52 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
  53 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
  54
  55 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
  56 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
  57                                   const object::ObjectFile &Obj,
  58                                   ProfCorrelatorKind FileKind) {
  59   auto C = std::make_unique<Context>();
  60   auto CountersSection = getInstrProfSection(Obj, IPSK_cnts);
  61   if (auto Err = CountersSection.takeError())
  62     return std::move(Err);
  63   if (FileKind == InstrProfCorrelator::BINARY) {
  64     auto DataSection = getInstrProfSection(Obj, IPSK_covdata);
  65     if (auto Err = DataSection.takeError())
  66       return std::move(Err);
  67     auto DataOrErr = DataSection->getContents();
  68     if (!DataOrErr)
  69       return DataOrErr.takeError();
  70     auto NameSection = getInstrProfSection(Obj, IPSK_covname);
  71     if (auto Err = NameSection.takeError())
  72       return std::move(Err);
  73     auto NameOrErr = NameSection->getContents();
  74     if (!NameOrErr)
  75       return NameOrErr.takeError();
  76     C->DataStart = DataOrErr->data();
  77     C->DataEnd = DataOrErr->data() + DataOrErr->size();
  78     C->NameStart = NameOrErr->data();
  79     C->NameSize = NameOrErr->size();
  80   }
  81   C->Buffer = std::move(Buffer);
  82   C->CountersSectionStart = CountersSection->getAddress();
  83   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
  84   // In COFF object file, there's a null byte at the beginning of the counter
  85   // section which doesn't exist in raw profile.
  86   if (Obj.getTripleObjectFormat() == Triple::COFF)
  87     ++C->CountersSectionStart;
  88
  89   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
  90   return Expected<std::unique_ptr<Context>>(std::move(C));
  91 }
  92
  93 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
  94 InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) {
  95   if (FileKind == DEBUG_INFO) {
  96     auto DsymObjectsOrErr =
  97         object::MachOObjectFile::findDsymObjectMembers(Filename);
  98     if (auto Err = DsymObjectsOrErr.takeError())
  99       return std::move(Err);
 100     if (!DsymObjectsOrErr->empty()) {
 101       // TODO: Enable profile correlation when there are multiple objects in a
 102       // dSYM bundle.
 103       if (DsymObjectsOrErr->size() > 1)
 104         return make_error<InstrProfError>(
 105             instrprof_error::unable_to_correlate_profile,
 106             "using multiple objects is not yet supported");
 107       Filename = *DsymObjectsOrErr->begin();
 108     }
 109     auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));
 110     if (auto Err = BufferOrErr.takeError())
 111       return std::move(Err);
 112
 113     return get(std::move(*BufferOrErr), FileKind);
 114   }
 115   if (FileKind == BINARY) {
 116     auto BufferOrErr = errorOrToExpected(MemoryBuffer::getFile(Filename));
 117     if (auto Err = BufferOrErr.takeError())
 118       return std::move(Err);
 119
 120     return get(std::move(*BufferOrErr), FileKind);
 121   }
 122   return make_error<InstrProfError>(
 123       instrprof_error::unable_to_correlate_profile,
 124       "unsupported correlation kind (only DWARF debug info and Binary format "
 125       "(ELF/COFF) are supported)");
 126 }
 127
 128 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
 129 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer,
 130                          ProfCorrelatorKind FileKind) {
 131   auto BinOrErr = object::createBinary(*Buffer);
 132   if (auto Err = BinOrErr.takeError())
 133     return std::move(Err);
 134
 135   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
 136     auto CtxOrErr = Context::get(std::move(Buffer), *Obj, FileKind);
 137     if (auto Err = CtxOrErr.takeError())
 138       return std::move(Err);
 139     auto T = Obj->makeTriple();
 140     if (T.isArch64Bit())
 141       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj,
 142                                                     FileKind);
 143     if (T.isArch32Bit())
 144       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj,
 145                                                     FileKind);
 146   }
 147   return make_error<InstrProfError>(
 148       instrprof_error::unable_to_correlate_profile, "not an object file");
 149 }
 150
 151 std::optional<size_t> InstrProfCorrelator::getDataSize() const {
 152   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
 153     return C->getDataSize();
 154   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
 155     return C->getDataSize();
 156   }
 157   return {};
 158 }
 159
 160 namespace llvm {
 161
 162 template <>
 163 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
 164     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 165     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
 166                               std::move(Ctx)) {}
 167 template <>
 168 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
 169     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
 170     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
 171                               std::move(Ctx)) {}
 172 template <>
 173 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
 174   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
 175 }
 176 template <>
 177 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
 178   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
 179 }
 180
 181 } // end namespace llvm
 182
 183 template <class IntPtrT>
 184 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
 185 InstrProfCorrelatorImpl<IntPtrT>::get(
 186     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
 187     const object::ObjectFile &Obj, ProfCorrelatorKind FileKind) {
 188   if (FileKind == DEBUG_INFO) {
 189     if (Obj.isELF() || Obj.isMachO()) {
 190       auto DICtx = DWARFContext::create(Obj);
 191       return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(
 192           std::move(DICtx), std::move(Ctx));
 193     }
 194     return make_error<InstrProfError>(
 195         instrprof_error::unable_to_correlate_profile,
 196         "unsupported debug info format (only DWARF is supported)");
 197   }
 198   if (Obj.isELF() || Obj.isCOFF())
 199     return std::make_unique<BinaryInstrProfCorrelator<IntPtrT>>(std::move(Ctx));
 200   return make_error<InstrProfError>(
 201       instrprof_error::unable_to_correlate_profile,
 202       "unsupported binary format (only ELF and COFF are supported)");
 203 }
 204
 205 template <class IntPtrT>
 206 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData(int MaxWarnings) {
 207   assert(Data.empty() && Names.empty() && NamesVec.empty());
 208   correlateProfileDataImpl(MaxWarnings);
 209   if (this->Data.empty())
 210     return make_error<InstrProfError>(
 211         instrprof_error::unable_to_correlate_profile,
 212         "could not find any profile data metadata in correlated file");
 213   Error Result = correlateProfileNameImpl();
 214   this->CounterOffsets.clear();
 215   this->NamesVec.clear();
 216   return Result;
 217 }
 218
 219 template <> struct yaml::MappingTraits<InstrProfCorrelator::CorrelationData> {
 220   static void mapping(yaml::IO &io,
 221                       InstrProfCorrelator::CorrelationData &Data) {
 222     io.mapRequired("Probes", Data.Probes);
 223   }
 224 };
 225
 226 template <> struct yaml::MappingTraits<InstrProfCorrelator::Probe> {
 227   static void mapping(yaml::IO &io, InstrProfCorrelator::Probe &P) {
 228     io.mapRequired("Function Name", P.FunctionName);
 229     io.mapOptional("Linkage Name", P.LinkageName);
 230     io.mapRequired("CFG Hash", P.CFGHash);
 231     io.mapRequired("Counter Offset", P.CounterOffset);
 232     io.mapRequired("Num Counters", P.NumCounters);
 233     io.mapOptional("File", P.FilePath);
 234     io.mapOptional("Line", P.LineNumber);
 235   }
 236 };
 237
 238 template <> struct yaml::SequenceElementTraits<InstrProfCorrelator::Probe> {
 239   static const bool flow = false;
 240 };
 241
 242 template <class IntPtrT>
 243 Error InstrProfCorrelatorImpl<IntPtrT>::dumpYaml(int MaxWarnings,
 244                                                  raw_ostream &OS) {
 245   InstrProfCorrelator::CorrelationData Data;
 246   correlateProfileDataImpl(MaxWarnings, &Data);
 247   if (Data.Probes.empty())
 248     return make_error<InstrProfError>(
 249         instrprof_error::unable_to_correlate_profile,
 250         "could not find any profile data metadata in debug info");
 251   yaml::Output YamlOS(OS);
 252   YamlOS << Data;
 253   return Error::success();
 254 }
 255
 256 template <class IntPtrT>
 257 void InstrProfCorrelatorImpl<IntPtrT>::addDataProbe(uint64_t NameRef,
 258                                                     uint64_t CFGHash,
 259                                                     IntPtrT CounterOffset,
 260                                                     IntPtrT FunctionPtr,
 261                                                     uint32_t NumCounters) {
 262   // Check if a probe was already added for this counter offset.
 263   if (!CounterOffsets.insert(CounterOffset).second)
 264     return;
 265   Data.push_back({
 266       maybeSwap<uint64_t>(NameRef),
 267       maybeSwap<uint64_t>(CFGHash),
 268       // In this mode, CounterPtr actually stores the section relative address
 269       // of the counter.
 270       maybeSwap<IntPtrT>(CounterOffset),
 271       // TODO: MC/DC is not yet supported.
 272       /*BitmapOffset=*/maybeSwap<IntPtrT>(0),
 273       maybeSwap<IntPtrT>(FunctionPtr),
 274       // TODO: Value profiling is not yet supported.
 275       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
 276       maybeSwap<uint32_t>(NumCounters),
 277       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
 278       // TODO: MC/DC is not yet supported.
 279       /*NumBitmapBytes=*/maybeSwap<uint32_t>(0),
 280   });
 281 }
 282
 283 template <class IntPtrT>
 284 std::optional<uint64_t>
 285 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
 286   auto Locations = Die.getLocations(dwarf::DW_AT_location);
 287   if (!Locations) {
 288     consumeError(Locations.takeError());
 289     return {};
 290   }
 291   auto &DU = *Die.getDwarfUnit();
 292   auto AddressSize = DU.getAddressByteSize();
 293   for (auto &Location : *Locations) {
 294     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
 295     DWARFExpression Expr(Data, AddressSize);
 296     for (auto &Op : Expr) {
 297       if (Op.getCode() == dwarf::DW_OP_addr) {
 298         return Op.getRawOperand(0);
 299       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
 300         uint64_t Index = Op.getRawOperand(0);
 301         if (auto SA = DU.getAddrOffsetSectionItem(Index))
 302           return SA->Address;
 303       }
 304     }
 305   }
 306   return {};
 307 }
 308
 309 template <class IntPtrT>
 310 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
 311   const auto &ParentDie = Die.getParent();
 312   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
 313     return false;
 314   if (Die.getTag() != dwarf::DW_TAG_variable)
 315     return false;
 316   if (!ParentDie.isSubprogramDIE())
 317     return false;
 318   if (!Die.hasChildren())
 319     return false;
 320   if (const char *Name = Die.getName(DINameKind::ShortName))
 321     return StringRef(Name).starts_with(getInstrProfCountersVarPrefix());
 322   return false;
 323 }
 324
 325 template <class IntPtrT>
 326 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
 327     int MaxWarnings, InstrProfCorrelator::CorrelationData *Data) {
 328   bool UnlimitedWarnings = (MaxWarnings == 0);
 329   // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
 330   int NumSuppressedWarnings = -MaxWarnings;
 331   auto maybeAddProbe = [&](DWARFDie Die) {
 332     if (!isDIEOfProbe(Die))
 333       return;
 334     std::optional<const char *> FunctionName;
 335     std::optional<uint64_t> CFGHash;
 336     std::optional<uint64_t> CounterPtr = getLocation(Die);
 337     auto FnDie = Die.getParent();
 338     auto FunctionPtr = dwarf::toAddress(FnDie.find(dwarf::DW_AT_low_pc));
 339     std::optional<uint64_t> NumCounters;
 340     for (const DWARFDie &Child : Die.children()) {
 341       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
 342         continue;
 343       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
 344       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
 345       if (!AnnotationFormName || !AnnotationFormValue)
 346         continue;
 347       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
 348       if (auto Err = AnnotationNameOrErr.takeError()) {
 349         consumeError(std::move(Err));
 350         continue;
 351       }
 352       StringRef AnnotationName = *AnnotationNameOrErr;
 353       if (AnnotationName.compare(
 354               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
 355         if (auto EC =
 356                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
 357           consumeError(std::move(EC));
 358       } else if (AnnotationName.compare(
 359                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
 360         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
 361       } else if (AnnotationName.compare(
 362                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
 363         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
 364       }
 365     }
 366     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
 367       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 368         WithColor::warning()
 369             << "Incomplete DIE for function " << FunctionName
 370             << ": CFGHash=" << CFGHash << "  CounterPtr=" << CounterPtr
 371             << "  NumCounters=" << NumCounters << "\n";
 372         LLVM_DEBUG(Die.dump(dbgs()));
 373       }
 374       return;
 375     }
 376     uint64_t CountersStart = this->Ctx->CountersSectionStart;
 377     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
 378     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
 379       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 380         WithColor::warning()
 381             << format("CounterPtr out of range for function %s: Actual=0x%x "
 382                       "Expected=[0x%x, 0x%x)\n",
 383                       *FunctionName, *CounterPtr, CountersStart, CountersEnd);
 384         LLVM_DEBUG(Die.dump(dbgs()));
 385       }
 386       return;
 387     }
 388     if (!FunctionPtr && (UnlimitedWarnings || ++NumSuppressedWarnings < 1)) {
 389       WithColor::warning() << format("Could not find address of function %s\n",
 390                                      *FunctionName);
 391       LLVM_DEBUG(Die.dump(dbgs()));
 392     }
 393     // In debug info correlation mode, the CounterPtr is an absolute address of
 394     // the counter, but it's expected to be relative later when iterating Data.
 395     IntPtrT CounterOffset = *CounterPtr - CountersStart;
 396     if (Data) {
 397       InstrProfCorrelator::Probe P;
 398       P.FunctionName = *FunctionName;
 399       if (auto Name = FnDie.getName(DINameKind::LinkageName))
 400         P.LinkageName = Name;
 401       P.CFGHash = *CFGHash;
 402       P.CounterOffset = CounterOffset;
 403       P.NumCounters = *NumCounters;
 404       auto FilePath = FnDie.getDeclFile(
 405           DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath);
 406       if (!FilePath.empty())
 407         P.FilePath = FilePath;
 408       if (auto LineNumber = FnDie.getDeclLine())
 409         P.LineNumber = LineNumber;
 410       Data->Probes.push_back(P);
 411     } else {
 412       this->addDataProbe(IndexedInstrProf::ComputeHash(*FunctionName), *CFGHash,
 413                          CounterOffset, FunctionPtr.value_or(0), *NumCounters);
 414       this->NamesVec.push_back(*FunctionName);
 415     }
 416   };
 417   for (auto &CU : DICtx->normal_units())
 418     for (const auto &Entry : CU->dies())
 419       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 420   for (auto &CU : DICtx->dwo_units())
 421     for (const auto &Entry : CU->dies())
 422       maybeAddProbe(DWARFDie(CU.get(), &Entry));
 423
 424   if (!UnlimitedWarnings && NumSuppressedWarnings > 0)
 425     WithColor::warning() << format("Suppressed %d additional warnings\n",
 426                                    NumSuppressedWarnings);
 427 }
 428
 429 template <class IntPtrT>
 430 Error DwarfInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
 431   if (this->NamesVec.empty()) {
 432     return make_error<InstrProfError>(
 433         instrprof_error::unable_to_correlate_profile,
 434         "could not find any profile name metadata in debug info");
 435   }
 436   auto Result =
 437       collectGlobalObjectNameStrings(this->NamesVec,
 438                                      /*doCompression=*/false, this->Names);
 439   return Result;
 440 }
 441
 442 template <class IntPtrT>
 443 void BinaryInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl(
 444     int MaxWarnings, InstrProfCorrelator::CorrelationData *CorrelateData) {
 445   using RawProfData = RawInstrProf::ProfileData<IntPtrT>;
 446   bool UnlimitedWarnings = (MaxWarnings == 0);
 447   // -N suppressed warnings means we can emit up to N (unsuppressed) warnings
 448   int NumSuppressedWarnings = -MaxWarnings;
 449
 450   const RawProfData *DataStart = (const RawProfData *)this->Ctx->DataStart;
 451   const RawProfData *DataEnd = (const RawProfData *)this->Ctx->DataEnd;
 452   // We need to use < here because the last data record may have no padding.
 453   for (const RawProfData *I = DataStart; I < DataEnd; ++I) {
 454     uint64_t CounterPtr = this->template maybeSwap<IntPtrT>(I->CounterPtr);
 455     uint64_t CountersStart = this->Ctx->CountersSectionStart;
 456     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
 457     if (CounterPtr < CountersStart || CounterPtr >= CountersEnd) {
 458       if (UnlimitedWarnings || ++NumSuppressedWarnings < 1) {
 459         WithColor::warning()
 460             << format("CounterPtr out of range for function: Actual=0x%x "
 461                       "Expected=[0x%x, 0x%x) at data offset=0x%x\n",
 462                       CounterPtr, CountersStart, CountersEnd,
 463                       (I - DataStart) * sizeof(RawProfData));
 464       }
 465     }
 466     // In binary correlation mode, the CounterPtr is an absolute address of the
 467     // counter, but it's expected to be relative later when iterating Data.
 468     IntPtrT CounterOffset = CounterPtr - CountersStart;
 469     this->addDataProbe(I->NameRef, I->FuncHash, CounterOffset,
 470                        I->FunctionPointer, I->NumCounters);
 471   }
 472 }
 473
 474 template <class IntPtrT>
 475 Error BinaryInstrProfCorrelator<IntPtrT>::correlateProfileNameImpl() {
 476   if (this->Ctx->NameSize == 0) {
 477     return make_error<InstrProfError>(
 478         instrprof_error::unable_to_correlate_profile,
 479         "could not find any profile data metadata in object file");
 480   }
 481   this->Names.append(this->Ctx->NameStart, this->Ctx->NameSize);
 482   return Error::success();
 483 }