bolt/lib/Profile/YAMLProfileReader.cpp

   1 //===- bolt/Profile/YAMLProfileReader.cpp - YAML profile de-serializer ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "bolt/Profile/YAMLProfileReader.h"
  10 #include "bolt/Core/BinaryBasicBlock.h"
  11 #include "bolt/Core/BinaryFunction.h"
  12 #include "bolt/Passes/MCF.h"
  13 #include "bolt/Profile/ProfileYAMLMapping.h"
  14 #include "bolt/Utils/Utils.h"
  15 #include "llvm/ADT/STLExtras.h"
  16 #include "llvm/Support/CommandLine.h"
  17
  18 using namespace llvm;
  19
  20 namespace opts {
  21
  22 extern cl::opt<unsigned> Verbosity;
  23 extern cl::OptionCategory BoltOptCategory;
  24 extern cl::opt<bool> InferStaleProfile;
  25
  26 static llvm::cl::opt<bool>
  27     IgnoreHash("profile-ignore-hash",
  28                cl::desc("ignore hash while reading function profile"),
  29                cl::Hidden, cl::cat(BoltOptCategory));
  30
  31 llvm::cl::opt<bool> ProfileUseDFS("profile-use-dfs",
  32                                   cl::desc("use DFS order for YAML profile"),
  33                                   cl::Hidden, cl::cat(BoltOptCategory));
  34 }
  35
  36 namespace llvm {
  37 namespace bolt {
  38
  39 bool YAMLProfileReader::isYAML(const StringRef Filename) {
  40   if (auto MB = MemoryBuffer::getFileOrSTDIN(Filename)) {
  41     StringRef Buffer = (*MB)->getBuffer();
  42     return Buffer.startswith("---\n");
  43   } else {
  44     report_error(Filename, MB.getError());
  45   }
  46   return false;
  47 }
  48
  49 void YAMLProfileReader::buildNameMaps(BinaryContext &BC) {
  50   auto lookupFunction = [&](StringRef Name) -> BinaryFunction * {
  51     if (BinaryData *BD = BC.getBinaryDataByName(Name))
  52       return BC.getFunctionForSymbol(BD->getSymbol());
  53     return nullptr;
  54   };
  55
  56   ProfileBFs.reserve(YamlBP.Functions.size());
  57
  58   for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
  59     StringRef Name = YamlBF.Name;
  60     const size_t Pos = Name.find("(*");
  61     if (Pos != StringRef::npos)
  62       Name = Name.substr(0, Pos);
  63     ProfileFunctionNames.insert(Name);
  64     ProfileBFs.push_back(lookupFunction(Name));
  65     if (const std::optional<StringRef> CommonName = getLTOCommonName(Name))
  66       LTOCommonNameMap[*CommonName].push_back(&YamlBF);
  67   }
  68   for (auto &[Symbol, BF] : BC.SymbolToFunctionMap) {
  69     StringRef Name = Symbol->getName();
  70     if (const std::optional<StringRef> CommonName = getLTOCommonName(Name))
  71       LTOCommonNameFunctionMap[*CommonName].insert(BF);
  72   }
  73 }
  74
  75 bool YAMLProfileReader::hasLocalsWithFileName() const {
  76   return llvm::any_of(ProfileFunctionNames.keys(), [](StringRef FuncName) {
  77     return FuncName.count('/') == 2 && FuncName[0] != '/';
  78   });
  79 }
  80
  81 bool YAMLProfileReader::parseFunctionProfile(
  82     BinaryFunction &BF, const yaml::bolt::BinaryFunctionProfile &YamlBF) {
  83   BinaryContext &BC = BF.getBinaryContext();
  84
  85   const bool IsDFSOrder = YamlBP.Header.IsDFSOrder;
  86   bool ProfileMatched = true;
  87   uint64_t MismatchedBlocks = 0;
  88   uint64_t MismatchedCalls = 0;
  89   uint64_t MismatchedEdges = 0;
  90
  91   uint64_t FunctionExecutionCount = 0;
  92
  93   BF.setExecutionCount(YamlBF.ExecCount);
  94
  95   uint64_t FuncRawBranchCount = 0;
  96   for (const yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks)
  97     for (const yaml::bolt::SuccessorInfo &YamlSI : YamlBB.Successors)
  98       FuncRawBranchCount += YamlSI.Count;
  99   BF.setRawBranchCount(FuncRawBranchCount);
 100
 101   if (!opts::IgnoreHash && YamlBF.Hash != BF.computeHash(IsDFSOrder)) {
 102     if (opts::Verbosity >= 1)
 103       errs() << "BOLT-WARNING: function hash mismatch\n";
 104     ProfileMatched = false;
 105   }
 106
 107   if (YamlBF.NumBasicBlocks != BF.size()) {
 108     if (opts::Verbosity >= 1)
 109       errs() << "BOLT-WARNING: number of basic blocks mismatch\n";
 110     ProfileMatched = false;
 111   }
 112
 113   BinaryFunction::BasicBlockOrderType Order;
 114   if (IsDFSOrder)
 115     llvm::copy(BF.dfs(), std::back_inserter(Order));
 116   else
 117     llvm::copy(BF.getLayout().blocks(), std::back_inserter(Order));
 118
 119   for (const yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks) {
 120     if (YamlBB.Index >= Order.size()) {
 121       if (opts::Verbosity >= 2)
 122         errs() << "BOLT-WARNING: index " << YamlBB.Index
 123                << " is out of bounds\n";
 124       ++MismatchedBlocks;
 125       continue;
 126     }
 127
 128     BinaryBasicBlock &BB = *Order[YamlBB.Index];
 129
 130     // Basic samples profile (without LBR) does not have branches information
 131     // and needs a special processing.
 132     if (YamlBP.Header.Flags & BinaryFunction::PF_SAMPLE) {
 133       if (!YamlBB.EventCount) {
 134         BB.setExecutionCount(0);
 135         continue;
 136       }
 137       uint64_t NumSamples = YamlBB.EventCount * 1000;
 138       if (NormalizeByInsnCount && BB.getNumNonPseudos())
 139         NumSamples /= BB.getNumNonPseudos();
 140       else if (NormalizeByCalls)
 141         NumSamples /= BB.getNumCalls() + 1;
 142
 143       BB.setExecutionCount(NumSamples);
 144       if (BB.isEntryPoint())
 145         FunctionExecutionCount += NumSamples;
 146       continue;
 147     }
 148
 149     BB.setExecutionCount(YamlBB.ExecCount);
 150
 151     for (const yaml::bolt::CallSiteInfo &YamlCSI : YamlBB.CallSites) {
 152       BinaryFunction *Callee = YamlCSI.DestId < YamlProfileToFunction.size()
 153                                    ? YamlProfileToFunction[YamlCSI.DestId]
 154                                    : nullptr;
 155       bool IsFunction = Callee ? true : false;
 156       MCSymbol *CalleeSymbol = nullptr;
 157       if (IsFunction)
 158         CalleeSymbol = Callee->getSymbolForEntryID(YamlCSI.EntryDiscriminator);
 159
 160       BF.getAllCallSites().emplace_back(CalleeSymbol, YamlCSI.Count,
 161                                         YamlCSI.Mispreds, YamlCSI.Offset);
 162
 163       if (YamlCSI.Offset >= BB.getOriginalSize()) {
 164         if (opts::Verbosity >= 2)
 165           errs() << "BOLT-WARNING: offset " << YamlCSI.Offset
 166                  << " out of bounds in block " << BB.getName() << '\n';
 167         ++MismatchedCalls;
 168         continue;
 169       }
 170
 171       MCInst *Instr =
 172           BF.getInstructionAtOffset(BB.getInputOffset() + YamlCSI.Offset);
 173       if (!Instr) {
 174         if (opts::Verbosity >= 2)
 175           errs() << "BOLT-WARNING: no instruction at offset " << YamlCSI.Offset
 176                  << " in block " << BB.getName() << '\n';
 177         ++MismatchedCalls;
 178         continue;
 179       }
 180       if (!BC.MIB->isCall(*Instr) && !BC.MIB->isIndirectBranch(*Instr)) {
 181         if (opts::Verbosity >= 2)
 182           errs() << "BOLT-WARNING: expected call at offset " << YamlCSI.Offset
 183                  << " in block " << BB.getName() << '\n';
 184         ++MismatchedCalls;
 185         continue;
 186       }
 187
 188       auto setAnnotation = [&](StringRef Name, uint64_t Count) {
 189         if (BC.MIB->hasAnnotation(*Instr, Name)) {
 190           if (opts::Verbosity >= 1)
 191             errs() << "BOLT-WARNING: ignoring duplicate " << Name
 192                    << " info for offset 0x" << Twine::utohexstr(YamlCSI.Offset)
 193                    << " in function " << BF << '\n';
 194           return;
 195         }
 196         BC.MIB->addAnnotation(*Instr, Name, Count);
 197       };
 198
 199       if (BC.MIB->isIndirectCall(*Instr) || BC.MIB->isIndirectBranch(*Instr)) {
 200         auto &CSP = BC.MIB->getOrCreateAnnotationAs<IndirectCallSiteProfile>(
 201             *Instr, "CallProfile");
 202         CSP.emplace_back(CalleeSymbol, YamlCSI.Count, YamlCSI.Mispreds);
 203       } else if (BC.MIB->getConditionalTailCall(*Instr)) {
 204         setAnnotation("CTCTakenCount", YamlCSI.Count);
 205         setAnnotation("CTCMispredCount", YamlCSI.Mispreds);
 206       } else {
 207         setAnnotation("Count", YamlCSI.Count);
 208       }
 209     }
 210
 211     for (const yaml::bolt::SuccessorInfo &YamlSI : YamlBB.Successors) {
 212       if (YamlSI.Index >= Order.size()) {
 213         if (opts::Verbosity >= 1)
 214           errs() << "BOLT-WARNING: index out of bounds for profiled block\n";
 215         ++MismatchedEdges;
 216         continue;
 217       }
 218
 219       BinaryBasicBlock &SuccessorBB = *Order[YamlSI.Index];
 220       if (!BB.getSuccessor(SuccessorBB.getLabel())) {
 221         if (opts::Verbosity >= 1)
 222           errs() << "BOLT-WARNING: no successor for block " << BB.getName()
 223                  << " that matches index " << YamlSI.Index << " or block "
 224                  << SuccessorBB.getName() << '\n';
 225         ++MismatchedEdges;
 226         continue;
 227       }
 228
 229       BinaryBasicBlock::BinaryBranchInfo &BI = BB.getBranchInfo(SuccessorBB);
 230       BI.Count += YamlSI.Count;
 231       BI.MispredictedCount += YamlSI.Mispreds;
 232     }
 233   }
 234
 235   // If basic block profile wasn't read it should be 0.
 236   for (BinaryBasicBlock &BB : BF)
 237     if (BB.getExecutionCount() == BinaryBasicBlock::COUNT_NO_PROFILE)
 238       BB.setExecutionCount(0);
 239
 240   if (YamlBP.Header.Flags & BinaryFunction::PF_SAMPLE) {
 241     BF.setExecutionCount(FunctionExecutionCount);
 242     estimateEdgeCounts(BF);
 243   }
 244
 245   ProfileMatched &= !MismatchedBlocks && !MismatchedCalls && !MismatchedEdges;
 246
 247   if (ProfileMatched)
 248     BF.markProfiled(YamlBP.Header.Flags);
 249
 250   if (!ProfileMatched && opts::Verbosity >= 1)
 251     errs() << "BOLT-WARNING: " << MismatchedBlocks << " blocks, "
 252            << MismatchedCalls << " calls, and " << MismatchedEdges
 253            << " edges in profile did not match function " << BF << '\n';
 254
 255   if (!ProfileMatched && opts::InferStaleProfile) {
 256     if (inferStaleProfile(BF, YamlBF)) {
 257       ProfileMatched = true;
 258       BF.markProfiled(YamlBP.Header.Flags);
 259     }
 260   }
 261
 262   return ProfileMatched;
 263 }
 264
 265 Error YAMLProfileReader::preprocessProfile(BinaryContext &BC) {
 266   ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
 267       MemoryBuffer::getFileOrSTDIN(Filename);
 268   if (std::error_code EC = MB.getError()) {
 269     errs() << "ERROR: cannot open " << Filename << ": " << EC.message() << "\n";
 270     return errorCodeToError(EC);
 271   }
 272   yaml::Input YamlInput(MB.get()->getBuffer());
 273
 274   // Consume YAML file.
 275   YamlInput >> YamlBP;
 276   if (YamlInput.error()) {
 277     errs() << "BOLT-ERROR: syntax error parsing profile in " << Filename
 278            << " : " << YamlInput.error().message() << '\n';
 279     return errorCodeToError(YamlInput.error());
 280   }
 281
 282   // Sanity check.
 283   if (YamlBP.Header.Version != 1)
 284     return make_error<StringError>(
 285         Twine("cannot read profile : unsupported version"),
 286         inconvertibleErrorCode());
 287
 288   if (YamlBP.Header.EventNames.find(',') != StringRef::npos)
 289     return make_error<StringError>(
 290         Twine("multiple events in profile are not supported"),
 291         inconvertibleErrorCode());
 292
 293   // Match profile to function based on a function name.
 294   buildNameMaps(BC);
 295
 296   // Preliminary assign function execution count.
 297   for (auto [YamlBF, BF] : llvm::zip_equal(YamlBP.Functions, ProfileBFs)) {
 298     if (!BF)
 299       continue;
 300     if (!BF->hasProfile()) {
 301       BF->setExecutionCount(YamlBF.ExecCount);
 302     } else {
 303       if (opts::Verbosity >= 1) {
 304         errs() << "BOLT-WARNING: dropping duplicate profile for " << YamlBF.Name
 305                << '\n';
 306       }
 307       BF = nullptr;
 308     }
 309   }
 310
 311   return Error::success();
 312 }
 313
 314 bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) {
 315   for (StringRef Name : BF.getNames())
 316     if (ProfileFunctionNames.contains(Name))
 317       return true;
 318   for (StringRef Name : BF.getNames()) {
 319     if (const std::optional<StringRef> CommonName = getLTOCommonName(Name)) {
 320       if (LTOCommonNameMap.contains(*CommonName))
 321         return true;
 322     }
 323   }
 324
 325   return false;
 326 }
 327
 328 Error YAMLProfileReader::readProfile(BinaryContext &BC) {
 329   YamlProfileToFunction.resize(YamlBP.Functions.size() + 1);
 330
 331   auto profileMatches = [](const yaml::bolt::BinaryFunctionProfile &Profile,
 332                            BinaryFunction &BF) {
 333     if (opts::IgnoreHash)
 334       return Profile.NumBasicBlocks == BF.size();
 335     return Profile.Hash == static_cast<uint64_t>(BF.getHash());
 336   };
 337
 338   // We have to do 2 passes since LTO introduces an ambiguity in function
 339   // names. The first pass assigns profiles that match 100% by name and
 340   // by hash. The second pass allows name ambiguity for LTO private functions.
 341   for (auto [YamlBF, BF] : llvm::zip_equal(YamlBP.Functions, ProfileBFs)) {
 342     if (!BF)
 343       continue;
 344     BinaryFunction &Function = *BF;
 345     // Clear function call count that may have been set while pre-processing
 346     // the profile.
 347     Function.setExecutionCount(BinaryFunction::COUNT_NO_PROFILE);
 348
 349     // Recompute hash once per function.
 350     if (!opts::IgnoreHash)
 351       Function.computeHash(YamlBP.Header.IsDFSOrder);
 352
 353     if (profileMatches(YamlBF, Function))
 354       matchProfileToFunction(YamlBF, Function);
 355   }
 356
 357   for (auto &[CommonName, LTOProfiles]: LTOCommonNameMap) {
 358     if (!LTOCommonNameFunctionMap.contains(CommonName))
 359       continue;
 360     std::unordered_set<BinaryFunction *> &Functions =
 361         LTOCommonNameFunctionMap[CommonName];
 362     // Return true if a given profile is matched to one of BinaryFunctions with
 363     // matching LTO common name.
 364     auto matchProfile = [&](yaml::bolt::BinaryFunctionProfile *YamlBF) {
 365       if (YamlBF->Used)
 366         return false;
 367       for (BinaryFunction *BF : Functions) {
 368         if (!ProfiledFunctions.count(BF) && profileMatches(*YamlBF, *BF)) {
 369           matchProfileToFunction(*YamlBF, *BF);
 370           return true;
 371         }
 372       }
 373       return false;
 374     };
 375     bool ProfileMatched = llvm::any_of(LTOProfiles, matchProfile);
 376
 377     // If there's only one function with a given name, try to match it
 378     // partially.
 379     if (!ProfileMatched && LTOProfiles.size() == 1 && Functions.size() == 1 &&
 380         !LTOProfiles.front()->Used &&
 381         !ProfiledFunctions.count(*Functions.begin()))
 382       matchProfileToFunction(*LTOProfiles.front(), **Functions.begin());
 383   }
 384
 385   for (auto [YamlBF, BF] : llvm::zip_equal(YamlBP.Functions, ProfileBFs))
 386     if (!YamlBF.Used && BF && !ProfiledFunctions.count(BF))
 387       matchProfileToFunction(YamlBF, *BF);
 388
 389   for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions)
 390     if (!YamlBF.Used && opts::Verbosity >= 1)
 391       errs() << "BOLT-WARNING: profile ignored for function " << YamlBF.Name
 392              << '\n';
 393
 394   // Set for parseFunctionProfile().
 395   NormalizeByInsnCount = usesEvent("cycles") || usesEvent("instructions");
 396   NormalizeByCalls = usesEvent("branches");
 397
 398   uint64_t NumUnused = 0;
 399   for (yaml::bolt::BinaryFunctionProfile &YamlBF : YamlBP.Functions) {
 400     if (YamlBF.Id >= YamlProfileToFunction.size()) {
 401       // Such profile was ignored.
 402       ++NumUnused;
 403       continue;
 404     }
 405     if (BinaryFunction *BF = YamlProfileToFunction[YamlBF.Id])
 406       parseFunctionProfile(*BF, YamlBF);
 407     else
 408       ++NumUnused;
 409   }
 410
 411   BC.setNumUnusedProfiledObjects(NumUnused);
 412
 413   return Error::success();
 414 }
 415
 416 bool YAMLProfileReader::usesEvent(StringRef Name) const {
 417   return YamlBP.Header.EventNames.find(std::string(Name)) != StringRef::npos;
 418 }
 419
 420 } // end namespace bolt
 421 } // end namespace llvm