llvm/lib/ProfileData/SampleProfReader.cpp

   1 //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the class that reads LLVM sample profiles. It
  10 // supports three file formats: text, binary and gcov.
  11 //
  12 // The textual representation is useful for debugging and testing purposes. The
  13 // binary representation is more compact, resulting in smaller file sizes.
  14 //
  15 // The gcov encoding is the one generated by GCC's AutoFDO profile creation
  16 // tool (https://github.com/google/autofdo)
  17 //
  18 // All three encodings can be used interchangeably as an input sample profile.
  19 //
  20 //===----------------------------------------------------------------------===//
  21
  22 #include "llvm/ProfileData/SampleProfReader.h"
  23 #include "llvm/ADT/DenseMap.h"
  24 #include "llvm/ADT/STLExtras.h"
  25 #include "llvm/ADT/StringRef.h"
  26 #include "llvm/IR/Module.h"
  27 #include "llvm/IR/ProfileSummary.h"
  28 #include "llvm/ProfileData/ProfileCommon.h"
  29 #include "llvm/ProfileData/SampleProf.h"
  30 #include "llvm/Support/CommandLine.h"
  31 #include "llvm/Support/Compression.h"
  32 #include "llvm/Support/ErrorOr.h"
  33 #include "llvm/Support/JSON.h"
  34 #include "llvm/Support/LEB128.h"
  35 #include "llvm/Support/LineIterator.h"
  36 #include "llvm/Support/MD5.h"
  37 #include "llvm/Support/MemoryBuffer.h"
  38 #include "llvm/Support/VirtualFileSystem.h"
  39 #include "llvm/Support/raw_ostream.h"
  40 #include <algorithm>
  41 #include <cstddef>
  42 #include <cstdint>
  43 #include <limits>
  44 #include <memory>
  45 #include <system_error>
  46 #include <vector>
  47
  48 using namespace llvm;
  49 using namespace sampleprof;
  50
  51 #define DEBUG_TYPE "samplepgo-reader"
  52
  53 // This internal option specifies if the profile uses FS discriminators.
  54 // It only applies to text, and binary format profiles.
  55 // For ext-binary format profiles, the flag is set in the summary.
  56 static cl::opt<bool> ProfileIsFSDisciminator(
  57     "profile-isfs", cl::Hidden, cl::init(false),
  58     cl::desc("Profile uses flow sensitive discriminators"));
  59
  60 /// Dump the function profile for \p FName.
  61 ///
  62 /// \param FContext Name + context of the function to print.
  63 /// \param OS Stream to emit the output to.
  64 void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,
  65                                               raw_ostream &OS) {
  66   OS << "Function: " << FS.getContext().toString() << ": " << FS;
  67 }
  68
  69 /// Dump all the function profiles found on stream \p OS.
  70 void SampleProfileReader::dump(raw_ostream &OS) {
  71   std::vector<NameFunctionSamples> V;
  72   sortFuncProfiles(Profiles, V);
  73   for (const auto &I : V)
  74     dumpFunctionProfile(*I.second, OS);
  75 }
  76
  77 static void dumpFunctionProfileJson(const FunctionSamples &S,
  78                                     json::OStream &JOS, bool TopLevel = false) {
  79   auto DumpBody = [&](const BodySampleMap &BodySamples) {
  80     for (const auto &I : BodySamples) {
  81       const LineLocation &Loc = I.first;
  82       const SampleRecord &Sample = I.second;
  83       JOS.object([&] {
  84         JOS.attribute("line", Loc.LineOffset);
  85         if (Loc.Discriminator)
  86           JOS.attribute("discriminator", Loc.Discriminator);
  87         JOS.attribute("samples", Sample.getSamples());
  88
  89         auto CallTargets = Sample.getSortedCallTargets();
  90         if (!CallTargets.empty()) {
  91           JOS.attributeArray("calls", [&] {
  92             for (const auto &J : CallTargets) {
  93               JOS.object([&] {
  94                 JOS.attribute("function", J.first);
  95                 JOS.attribute("samples", J.second);
  96               });
  97             }
  98           });
  99         }
 100       });
 101     }
 102   };
 103
 104   auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {
 105     for (const auto &I : CallsiteSamples)
 106       for (const auto &FS : I.second) {
 107         const LineLocation &Loc = I.first;
 108         const FunctionSamples &CalleeSamples = FS.second;
 109         JOS.object([&] {
 110           JOS.attribute("line", Loc.LineOffset);
 111           if (Loc.Discriminator)
 112             JOS.attribute("discriminator", Loc.Discriminator);
 113           JOS.attributeArray(
 114               "samples", [&] { dumpFunctionProfileJson(CalleeSamples, JOS); });
 115         });
 116       }
 117   };
 118
 119   JOS.object([&] {
 120     JOS.attribute("name", S.getName());
 121     JOS.attribute("total", S.getTotalSamples());
 122     if (TopLevel)
 123       JOS.attribute("head", S.getHeadSamples());
 124
 125     const auto &BodySamples = S.getBodySamples();
 126     if (!BodySamples.empty())
 127       JOS.attributeArray("body", [&] { DumpBody(BodySamples); });
 128
 129     const auto &CallsiteSamples = S.getCallsiteSamples();
 130     if (!CallsiteSamples.empty())
 131       JOS.attributeArray("callsites",
 132                          [&] { DumpCallsiteSamples(CallsiteSamples); });
 133   });
 134 }
 135
 136 /// Dump all the function profiles found on stream \p OS in the JSON format.
 137 void SampleProfileReader::dumpJson(raw_ostream &OS) {
 138   std::vector<NameFunctionSamples> V;
 139   sortFuncProfiles(Profiles, V);
 140   json::OStream JOS(OS, 2);
 141   JOS.arrayBegin();
 142   for (const auto &F : V)
 143     dumpFunctionProfileJson(*F.second, JOS, true);
 144   JOS.arrayEnd();
 145
 146   // Emit a newline character at the end as json::OStream doesn't emit one.
 147   OS << "\n";
 148 }
 149
 150 /// Parse \p Input as function head.
 151 ///
 152 /// Parse one line of \p Input, and update function name in \p FName,
 153 /// function's total sample count in \p NumSamples, function's entry
 154 /// count in \p NumHeadSamples.
 155 ///
 156 /// \returns true if parsing is successful.
 157 static bool ParseHead(const StringRef &Input, StringRef &FName,
 158                       uint64_t &NumSamples, uint64_t &NumHeadSamples) {
 159   if (Input[0] == ' ')
 160     return false;
 161   size_t n2 = Input.rfind(':');
 162   size_t n1 = Input.rfind(':', n2 - 1);
 163   FName = Input.substr(0, n1);
 164   if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))
 165     return false;
 166   if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))
 167     return false;
 168   return true;
 169 }
 170
 171 /// Returns true if line offset \p L is legal (only has 16 bits).
 172 static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }
 173
 174 /// Parse \p Input that contains metadata.
 175 /// Possible metadata:
 176 /// - CFG Checksum information:
 177 ///     !CFGChecksum: 12345
 178 /// - CFG Checksum information:
 179 ///     !Attributes: 1
 180 /// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.
 181 static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,
 182                           uint32_t &Attributes) {
 183   if (Input.startswith("!CFGChecksum:")) {
 184     StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();
 185     return !CFGInfo.getAsInteger(10, FunctionHash);
 186   }
 187
 188   if (Input.startswith("!Attributes:")) {
 189     StringRef Attrib = Input.substr(strlen("!Attributes:")).trim();
 190     return !Attrib.getAsInteger(10, Attributes);
 191   }
 192
 193   return false;
 194 }
 195
 196 enum class LineType {
 197   CallSiteProfile,
 198   BodyProfile,
 199   Metadata,
 200 };
 201
 202 /// Parse \p Input as line sample.
 203 ///
 204 /// \param Input input line.
 205 /// \param LineTy Type of this line.
 206 /// \param Depth the depth of the inline stack.
 207 /// \param NumSamples total samples of the line/inlined callsite.
 208 /// \param LineOffset line offset to the start of the function.
 209 /// \param Discriminator discriminator of the line.
 210 /// \param TargetCountMap map from indirect call target to count.
 211 /// \param FunctionHash the function's CFG hash, used by pseudo probe.
 212 ///
 213 /// returns true if parsing is successful.
 214 static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,
 215                       uint64_t &NumSamples, uint32_t &LineOffset,
 216                       uint32_t &Discriminator, StringRef &CalleeName,
 217                       DenseMap<StringRef, uint64_t> &TargetCountMap,
 218                       uint64_t &FunctionHash, uint32_t &Attributes) {
 219   for (Depth = 0; Input[Depth] == ' '; Depth++)
 220     ;
 221   if (Depth == 0)
 222     return false;
 223
 224   if (Input[Depth] == '!') {
 225     LineTy = LineType::Metadata;
 226     return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);
 227   }
 228
 229   size_t n1 = Input.find(':');
 230   StringRef Loc = Input.substr(Depth, n1 - Depth);
 231   size_t n2 = Loc.find('.');
 232   if (n2 == StringRef::npos) {
 233     if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset))
 234       return false;
 235     Discriminator = 0;
 236   } else {
 237     if (Loc.substr(0, n2).getAsInteger(10, LineOffset))
 238       return false;
 239     if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator))
 240       return false;
 241   }
 242
 243   StringRef Rest = Input.substr(n1 + 2);
 244   if (isDigit(Rest[0])) {
 245     LineTy = LineType::BodyProfile;
 246     size_t n3 = Rest.find(' ');
 247     if (n3 == StringRef::npos) {
 248       if (Rest.getAsInteger(10, NumSamples))
 249         return false;
 250     } else {
 251       if (Rest.substr(0, n3).getAsInteger(10, NumSamples))
 252         return false;
 253     }
 254     // Find call targets and their sample counts.
 255     // Note: In some cases, there are symbols in the profile which are not
 256     // mangled. To accommodate such cases, use colon + integer pairs as the
 257     // anchor points.
 258     // An example:
 259     // _M_construct<char *>:1000 string_view<std::allocator<char> >:437
 260     // ":1000" and ":437" are used as anchor points so the string above will
 261     // be interpreted as
 262     // target: _M_construct<char *>
 263     // count: 1000
 264     // target: string_view<std::allocator<char> >
 265     // count: 437
 266     while (n3 != StringRef::npos) {
 267       n3 += Rest.substr(n3).find_first_not_of(' ');
 268       Rest = Rest.substr(n3);
 269       n3 = Rest.find_first_of(':');
 270       if (n3 == StringRef::npos || n3 == 0)
 271         return false;
 272
 273       StringRef Target;
 274       uint64_t count, n4;
 275       while (true) {
 276         // Get the segment after the current colon.
 277         StringRef AfterColon = Rest.substr(n3 + 1);
 278         // Get the target symbol before the current colon.
 279         Target = Rest.substr(0, n3);
 280         // Check if the word after the current colon is an integer.
 281         n4 = AfterColon.find_first_of(' ');
 282         n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();
 283         StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);
 284         if (!WordAfterColon.getAsInteger(10, count))
 285           break;
 286
 287         // Try to find the next colon.
 288         uint64_t n5 = AfterColon.find_first_of(':');
 289         if (n5 == StringRef::npos)
 290           return false;
 291         n3 += n5 + 1;
 292       }
 293
 294       // An anchor point is found. Save the {target, count} pair
 295       TargetCountMap[Target] = count;
 296       if (n4 == Rest.size())
 297         break;
 298       // Change n3 to the next blank space after colon + integer pair.
 299       n3 = n4;
 300     }
 301   } else {
 302     LineTy = LineType::CallSiteProfile;
 303     size_t n3 = Rest.find_last_of(':');
 304     CalleeName = Rest.substr(0, n3);
 305     if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))
 306       return false;
 307   }
 308   return true;
 309 }
 310
 311 /// Load samples from a text file.
 312 ///
 313 /// See the documentation at the top of the file for an explanation of
 314 /// the expected format.
 315 ///
 316 /// \returns true if the file was loaded successfully, false otherwise.
 317 std::error_code SampleProfileReaderText::readImpl() {
 318   line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');
 319   sampleprof_error Result = sampleprof_error::success;
 320
 321   InlineCallStack InlineStack;
 322   uint32_t TopLevelProbeProfileCount = 0;
 323
 324   // DepthMetadata tracks whether we have processed metadata for the current
 325   // top-level or nested function profile.
 326   uint32_t DepthMetadata = 0;
 327
 328   ProfileIsFS = ProfileIsFSDisciminator;
 329   FunctionSamples::ProfileIsFS = ProfileIsFS;
 330   for (; !LineIt.is_at_eof(); ++LineIt) {
 331     size_t pos = LineIt->find_first_not_of(' ');
 332     if (pos == LineIt->npos || (*LineIt)[pos] == '#')
 333       continue;
 334     // Read the header of each function.
 335     //
 336     // Note that for function identifiers we are actually expecting
 337     // mangled names, but we may not always get them. This happens when
 338     // the compiler decides not to emit the function (e.g., it was inlined
 339     // and removed). In this case, the binary will not have the linkage
 340     // name for the function, so the profiler will emit the function's
 341     // unmangled name, which may contain characters like ':' and '>' in its
 342     // name (member functions, templates, etc).
 343     //
 344     // The only requirement we place on the identifier, then, is that it
 345     // should not begin with a number.
 346     if ((*LineIt)[0] != ' ') {
 347       uint64_t NumSamples, NumHeadSamples;
 348       StringRef FName;
 349       if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {
 350         reportError(LineIt.line_number(),
 351                     "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
 352         return sampleprof_error::malformed;
 353       }
 354       DepthMetadata = 0;
 355       SampleContext FContext(FName, CSNameTable);
 356       if (FContext.hasContext())
 357         ++CSProfileCount;
 358       FunctionSamples &FProfile = Profiles.Create(FContext);
 359       MergeResult(Result, FProfile.addTotalSamples(NumSamples));
 360       MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
 361       InlineStack.clear();
 362       InlineStack.push_back(&FProfile);
 363     } else {
 364       uint64_t NumSamples;
 365       StringRef FName;
 366       DenseMap<StringRef, uint64_t> TargetCountMap;
 367       uint32_t Depth, LineOffset, Discriminator;
 368       LineType LineTy;
 369       uint64_t FunctionHash = 0;
 370       uint32_t Attributes = 0;
 371       if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
 372                      Discriminator, FName, TargetCountMap, FunctionHash,
 373                      Attributes)) {
 374         reportError(LineIt.line_number(),
 375                     "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
 376                         *LineIt);
 377         return sampleprof_error::malformed;
 378       }
 379       if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
 380         // Metadata must be put at the end of a function profile.
 381         reportError(LineIt.line_number(),
 382                     "Found non-metadata after metadata: " + *LineIt);
 383         return sampleprof_error::malformed;
 384       }
 385
 386       // Here we handle FS discriminators.
 387       Discriminator &= getDiscriminatorMask();
 388
 389       while (InlineStack.size() > Depth) {
 390         InlineStack.pop_back();
 391       }
 392       switch (LineTy) {
 393       case LineType::CallSiteProfile: {
 394         FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
 395             LineLocation(LineOffset, Discriminator))[std::string(FName)];
 396         FSamples.setName(FName);
 397         MergeResult(Result, FSamples.addTotalSamples(NumSamples));
 398         InlineStack.push_back(&FSamples);
 399         DepthMetadata = 0;
 400         break;
 401       }
 402       case LineType::BodyProfile: {
 403         while (InlineStack.size() > Depth) {
 404           InlineStack.pop_back();
 405         }
 406         FunctionSamples &FProfile = *InlineStack.back();
 407         for (const auto &name_count : TargetCountMap) {
 408           MergeResult(Result, FProfile.addCalledTargetSamples(
 409                                   LineOffset, Discriminator, name_count.first,
 410                                   name_count.second));
 411         }
 412         MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
 413                                                     NumSamples));
 414         break;
 415       }
 416       case LineType::Metadata: {
 417         FunctionSamples &FProfile = *InlineStack.back();
 418         if (FunctionHash) {
 419           FProfile.setFunctionHash(FunctionHash);
 420           if (Depth == 1)
 421             ++TopLevelProbeProfileCount;
 422         }
 423         FProfile.getContext().setAllAttributes(Attributes);
 424         if (Attributes & (uint32_t)ContextShouldBeInlined)
 425           ProfileIsPreInlined = true;
 426         DepthMetadata = Depth;
 427         break;
 428       }
 429       }
 430     }
 431   }
 432
 433   assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
 434          "Cannot have both context-sensitive and regular profile");
 435   ProfileIsCS = (CSProfileCount > 0);
 436   assert((TopLevelProbeProfileCount == 0 ||
 437           TopLevelProbeProfileCount == Profiles.size()) &&
 438          "Cannot have both probe-based profiles and regular profiles");
 439   ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);
 440   FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
 441   FunctionSamples::ProfileIsCS = ProfileIsCS;
 442   FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;
 443
 444   if (Result == sampleprof_error::success)
 445     computeSummary();
 446
 447   return Result;
 448 }
 449
 450 bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {
 451   bool result = false;
 452
 453   // Check that the first non-comment line is a valid function header.
 454   line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');
 455   if (!LineIt.is_at_eof()) {
 456     if ((*LineIt)[0] != ' ') {
 457       uint64_t NumSamples, NumHeadSamples;
 458       StringRef FName;
 459       result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);
 460     }
 461   }
 462
 463   return result;
 464 }
 465
 466 template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {
 467   unsigned NumBytesRead = 0;
 468   uint64_t Val = decodeULEB128(Data, &NumBytesRead);
 469
 470   if (Val > std::numeric_limits<T>::max()) {
 471     std::error_code EC = sampleprof_error::malformed;
 472     reportError(0, EC.message());
 473     return EC;
 474   } else if (Data + NumBytesRead > End) {
 475     std::error_code EC = sampleprof_error::truncated;
 476     reportError(0, EC.message());
 477     return EC;
 478   }
 479
 480   Data += NumBytesRead;
 481   return static_cast<T>(Val);
 482 }
 483
 484 ErrorOr<StringRef> SampleProfileReaderBinary::readString() {
 485   StringRef Str(reinterpret_cast<const char *>(Data));
 486   if (Data + Str.size() + 1 > End) {
 487     std::error_code EC = sampleprof_error::truncated;
 488     reportError(0, EC.message());
 489     return EC;
 490   }
 491
 492   Data += Str.size() + 1;
 493   return Str;
 494 }
 495
 496 template <typename T>
 497 ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {
 498   if (Data + sizeof(T) > End) {
 499     std::error_code EC = sampleprof_error::truncated;
 500     reportError(0, EC.message());
 501     return EC;
 502   }
 503
 504   using namespace support;
 505   T Val = endian::readNext<T, little, unaligned>(Data);
 506   return Val;
 507 }
 508
 509 template <typename T>
 510 inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
 511   auto Idx = readNumber<size_t>();
 512   if (std::error_code EC = Idx.getError())
 513     return EC;
 514   if (*Idx >= Table.size())
 515     return sampleprof_error::truncated_name_table;
 516   return *Idx;
 517 }
 518
 519 ErrorOr<StringRef>
 520 SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
 521   auto Idx = readStringIndex(NameTable);
 522   if (std::error_code EC = Idx.getError())
 523     return EC;
 524
 525   // Lazy loading, if the string has not been materialized from memory storing
 526   // MD5 values, then it is default initialized with the null pointer. This can
 527   // only happen when using fixed length MD5, that bounds check is performed
 528   // while parsing the name table to ensure MD5NameMemStart points to an array
 529   // with enough MD5 entries.
 530   StringRef &SR = NameTable[*Idx];
 531   if (!SR.data()) {
 532     assert(MD5NameMemStart);
 533     using namespace support;
 534     uint64_t FID = endian::read<uint64_t, little, unaligned>(
 535        MD5NameMemStart + (*Idx) * sizeof(uint64_t));
 536     SR = MD5StringBuf.emplace_back(std::to_string(FID));
 537   }
 538   if (RetIdx)
 539     *RetIdx = *Idx;
 540   return SR;
 541 }
 542
 543 ErrorOr<SampleContextFrames>
 544 SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {
 545   auto ContextIdx = readNumber<size_t>();
 546   if (std::error_code EC = ContextIdx.getError())
 547     return EC;
 548   if (*ContextIdx >= CSNameTable.size())
 549     return sampleprof_error::truncated_name_table;
 550   if (RetIdx)
 551     *RetIdx = *ContextIdx;
 552   return CSNameTable[*ContextIdx];
 553 }
 554
 555 ErrorOr<std::pair<SampleContext, uint64_t>>
 556 SampleProfileReaderBinary::readSampleContextFromTable() {
 557   SampleContext Context;
 558   size_t Idx;
 559   if (ProfileIsCS) {
 560     auto FContext(readContextFromTable(&Idx));
 561     if (std::error_code EC = FContext.getError())
 562       return EC;
 563     Context = SampleContext(*FContext);
 564   } else {
 565     auto FName(readStringFromTable(&Idx));
 566     if (std::error_code EC = FName.getError())
 567       return EC;
 568     Context = SampleContext(*FName);
 569   }
 570   // Since MD5SampleContextStart may point to the profile's file data, need to
 571   // make sure it is reading the same value on big endian CPU.
 572   uint64_t Hash = support::endian::read64le(MD5SampleContextStart + Idx);
 573   // Lazy computing of hash value, write back to the table to cache it. Only
 574   // compute the context's hash value if it is being referenced for the first
 575   // time.
 576   if (Hash == 0) {
 577     assert(MD5SampleContextStart == MD5SampleContextTable.data());
 578     Hash = Context.getHashCode();
 579     support::endian::write64le(&MD5SampleContextTable[Idx], Hash);
 580   }
 581   return std::make_pair(Context, Hash);
 582 }
 583
 584 std::error_code
 585 SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
 586   auto NumSamples = readNumber<uint64_t>();
 587   if (std::error_code EC = NumSamples.getError())
 588     return EC;
 589   FProfile.addTotalSamples(*NumSamples);
 590
 591   // Read the samples in the body.
 592   auto NumRecords = readNumber<uint32_t>();
 593   if (std::error_code EC = NumRecords.getError())
 594     return EC;
 595
 596   for (uint32_t I = 0; I < *NumRecords; ++I) {
 597     auto LineOffset = readNumber<uint64_t>();
 598     if (std::error_code EC = LineOffset.getError())
 599       return EC;
 600
 601     if (!isOffsetLegal(*LineOffset)) {
 602       return std::error_code();
 603     }
 604
 605     auto Discriminator = readNumber<uint64_t>();
 606     if (std::error_code EC = Discriminator.getError())
 607       return EC;
 608
 609     auto NumSamples = readNumber<uint64_t>();
 610     if (std::error_code EC = NumSamples.getError())
 611       return EC;
 612
 613     auto NumCalls = readNumber<uint32_t>();
 614     if (std::error_code EC = NumCalls.getError())
 615       return EC;
 616
 617     // Here we handle FS discriminators:
 618     uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
 619
 620     for (uint32_t J = 0; J < *NumCalls; ++J) {
 621       auto CalledFunction(readStringFromTable());
 622       if (std::error_code EC = CalledFunction.getError())
 623         return EC;
 624
 625       auto CalledFunctionSamples = readNumber<uint64_t>();
 626       if (std::error_code EC = CalledFunctionSamples.getError())
 627         return EC;
 628
 629       FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal,
 630                                       *CalledFunction, *CalledFunctionSamples);
 631     }
 632
 633     FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples);
 634   }
 635
 636   // Read all the samples for inlined function calls.
 637   auto NumCallsites = readNumber<uint32_t>();
 638   if (std::error_code EC = NumCallsites.getError())
 639     return EC;
 640
 641   for (uint32_t J = 0; J < *NumCallsites; ++J) {
 642     auto LineOffset = readNumber<uint64_t>();
 643     if (std::error_code EC = LineOffset.getError())
 644       return EC;
 645
 646     auto Discriminator = readNumber<uint64_t>();
 647     if (std::error_code EC = Discriminator.getError())
 648       return EC;
 649
 650     auto FName(readStringFromTable());
 651     if (std::error_code EC = FName.getError())
 652       return EC;
 653
 654     // Here we handle FS discriminators:
 655     uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
 656
 657     FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
 658         LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)];
 659     CalleeProfile.setName(*FName);
 660     if (std::error_code EC = readProfile(CalleeProfile))
 661       return EC;
 662   }
 663
 664   return sampleprof_error::success;
 665 }
 666
 667 std::error_code
 668 SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
 669   Data = Start;
 670   auto NumHeadSamples = readNumber<uint64_t>();
 671   if (std::error_code EC = NumHeadSamples.getError())
 672     return EC;
 673
 674   auto FContextHash(readSampleContextFromTable());
 675   if (std::error_code EC = FContextHash.getError())
 676     return EC;
 677
 678   auto &[FContext, Hash] = *FContextHash;
 679   // Use the cached hash value for insertion instead of recalculating it.
 680   auto Res = Profiles.try_emplace(Hash, FContext, FunctionSamples());
 681   FunctionSamples &FProfile = Res.first->second;
 682   FProfile.setContext(FContext);
 683   FProfile.addHeadSamples(*NumHeadSamples);
 684
 685   if (FContext.hasContext())
 686     CSProfileCount++;
 687
 688   if (std::error_code EC = readProfile(FProfile))
 689     return EC;
 690   return sampleprof_error::success;
 691 }
 692
 693 std::error_code SampleProfileReaderBinary::readImpl() {
 694   ProfileIsFS = ProfileIsFSDisciminator;
 695   FunctionSamples::ProfileIsFS = ProfileIsFS;
 696   while (Data < End) {
 697     if (std::error_code EC = readFuncProfile(Data))
 698       return EC;
 699   }
 700
 701   return sampleprof_error::success;
 702 }
 703
 704 std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
 705     const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {
 706   Data = Start;
 707   End = Start + Size;
 708   switch (Entry.Type) {
 709   case SecProfSummary:
 710     if (std::error_code EC = readSummary())
 711       return EC;
 712     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
 713       Summary->setPartialProfile(true);
 714     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext))
 715       FunctionSamples::ProfileIsCS = ProfileIsCS = true;
 716     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined))
 717       FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;
 718     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))
 719       FunctionSamples::ProfileIsFS = ProfileIsFS = true;
 720     break;
 721   case SecNameTable: {
 722     bool FixedLengthMD5 =
 723         hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5);
 724     bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name);
 725     // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire
 726     // profile uses MD5 for function name matching in IPO passes.
 727     ProfileIsMD5 = ProfileIsMD5 || UseMD5;
 728     FunctionSamples::HasUniqSuffix =
 729         hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix);
 730     if (std::error_code EC = readNameTableSec(UseMD5, FixedLengthMD5))
 731       return EC;
 732     break;
 733   }
 734   case SecCSNameTable: {
 735     if (std::error_code EC = readCSNameTableSec())
 736       return EC;
 737     break;
 738   }
 739   case SecLBRProfile:
 740     if (std::error_code EC = readFuncProfiles())
 741       return EC;
 742     break;
 743   case SecFuncOffsetTable:
 744     // If module is absent, we are using LLVM tools, and need to read all
 745     // profiles, so skip reading the function offset table.
 746     if (!M) {
 747       Data = End;
 748     } else {
 749       assert((!ProfileIsCS ||
 750               hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&
 751              "func offset table should always be sorted in CS profile");
 752       if (std::error_code EC = readFuncOffsetTable())
 753         return EC;
 754     }
 755     break;
 756   case SecFuncMetadata: {
 757     ProfileIsProbeBased =
 758         hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
 759     FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
 760     bool HasAttribute =
 761         hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);
 762     if (std::error_code EC = readFuncMetadata(HasAttribute))
 763       return EC;
 764     break;
 765   }
 766   case SecProfileSymbolList:
 767     if (std::error_code EC = readProfileSymbolList())
 768       return EC;
 769     break;
 770   default:
 771     if (std::error_code EC = readCustomSection(Entry))
 772       return EC;
 773     break;
 774   }
 775   return sampleprof_error::success;
 776 }
 777
 778 bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
 779   // If profile is CS, the function offset section is expected to consist of
 780   // sequences of contexts in pre-order layout
 781   // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched
 782   // context in the module is found, the profiles of all its callees are
 783   // recursively loaded. A list is needed since the order of profiles matters.
 784   if (ProfileIsCS)
 785     return true;
 786
 787   // If the profile is MD5, use the map container to lookup functions in
 788   // the module. A remapper has no use on MD5 names.
 789   if (useMD5())
 790     return false;
 791
 792   // Profile is not MD5 and if a remapper is present, the remapped name of
 793   // every function needed to be matched against the module, so use the list
 794   // container since each entry is accessed.
 795   if (Remapper)
 796     return true;
 797
 798   // Otherwise use the map container for faster lookup.
 799   // TODO: If the cardinality of the function offset section is much smaller
 800   // than the number of functions in the module, using the list container can
 801   // be always faster, but we need to figure out the constant factor to
 802   // determine the cutoff.
 803   return false;
 804 }
 805
 806
 807 bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
 808   if (!M)
 809     return false;
 810   FuncsToUse.clear();
 811   for (auto &F : *M)
 812     FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
 813   return true;
 814 }
 815
 816 std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
 817   // If there are more than one function offset section, the profile associated
 818   // with the previous section has to be done reading before next one is read.
 819   FuncOffsetTable.clear();
 820   FuncOffsetList.clear();
 821
 822   auto Size = readNumber<uint64_t>();
 823   if (std::error_code EC = Size.getError())
 824     return EC;
 825
 826   bool UseFuncOffsetList = useFuncOffsetList();
 827   if (UseFuncOffsetList)
 828     FuncOffsetList.reserve(*Size);
 829   else
 830     FuncOffsetTable.reserve(*Size);
 831
 832   for (uint64_t I = 0; I < *Size; ++I) {
 833     auto FContextHash(readSampleContextFromTable());
 834     if (std::error_code EC = FContextHash.getError())
 835       return EC;
 836
 837     auto &[FContext, Hash] = *FContextHash;
 838     auto Offset = readNumber<uint64_t>();
 839     if (std::error_code EC = Offset.getError())
 840       return EC;
 841
 842     if (UseFuncOffsetList)
 843       FuncOffsetList.emplace_back(FContext, *Offset);
 844     else
 845       // Because Porfiles replace existing value with new value if collision
 846       // happens, we also use the latest offset so that they are consistent.
 847       FuncOffsetTable[Hash] = *Offset;
 848  }
 849
 850  return sampleprof_error::success;
 851 }
 852
 853 std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
 854   // Collect functions used by current module if the Reader has been
 855   // given a module.
 856   // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName
 857   // which will query FunctionSamples::HasUniqSuffix, so it has to be
 858   // called after FunctionSamples::HasUniqSuffix is set, i.e. after
 859   // NameTable section is read.
 860   bool LoadFuncsToBeUsed = collectFuncsFromModule();
 861
 862   // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all
 863   // profiles.
 864   const uint8_t *Start = Data;
 865   if (!LoadFuncsToBeUsed) {
 866     while (Data < End) {
 867       if (std::error_code EC = readFuncProfile(Data))
 868         return EC;
 869     }
 870     assert(Data == End && "More data is read than expected");
 871   } else {
 872     // Load function profiles on demand.
 873     if (Remapper) {
 874       for (auto Name : FuncsToUse) {
 875         Remapper->insert(Name);
 876       }
 877     }
 878
 879     if (ProfileIsCS) {
 880       assert(useFuncOffsetList());
 881       DenseSet<uint64_t> FuncGuidsToUse;
 882       if (useMD5()) {
 883         for (auto Name : FuncsToUse)
 884           FuncGuidsToUse.insert(Function::getGUID(Name));
 885       }
 886
 887       // For each function in current module, load all context profiles for
 888       // the function as well as their callee contexts which can help profile
 889       // guided importing for ThinLTO. This can be achieved by walking
 890       // through an ordered context container, where contexts are laid out
 891       // as if they were walked in preorder of a context trie. While
 892       // traversing the trie, a link to the highest common ancestor node is
 893       // kept so that all of its decendants will be loaded.
 894       const SampleContext *CommonContext = nullptr;
 895       for (const auto &NameOffset : FuncOffsetList) {
 896         const auto &FContext = NameOffset.first;
 897         auto FName = FContext.getName();
 898         // For function in the current module, keep its farthest ancestor
 899         // context. This can be used to load itself and its child and
 900         // sibling contexts.
 901         if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) ||
 902             (!useMD5() && (FuncsToUse.count(FName) ||
 903                            (Remapper && Remapper->exist(FName))))) {
 904           if (!CommonContext || !CommonContext->IsPrefixOf(FContext))
 905             CommonContext = &FContext;
 906         }
 907
 908         if (CommonContext == &FContext ||
 909             (CommonContext && CommonContext->IsPrefixOf(FContext))) {
 910           // Load profile for the current context which originated from
 911           // the common ancestor.
 912           const uint8_t *FuncProfileAddr = Start + NameOffset.second;
 913           if (std::error_code EC = readFuncProfile(FuncProfileAddr))
 914             return EC;
 915         }
 916       }
 917     } else if (useMD5()) {
 918       assert(!useFuncOffsetList());
 919       for (auto Name : FuncsToUse) {
 920         auto GUID = MD5Hash(Name);
 921         auto iter = FuncOffsetTable.find(GUID);
 922         if (iter == FuncOffsetTable.end())
 923           continue;
 924         const uint8_t *FuncProfileAddr = Start + iter->second;
 925         if (std::error_code EC = readFuncProfile(FuncProfileAddr))
 926           return EC;
 927       }
 928     } else if (Remapper) {
 929       assert(useFuncOffsetList());
 930       for (auto NameOffset : FuncOffsetList) {
 931         SampleContext FContext(NameOffset.first);
 932         auto FuncName = FContext.getName();
 933         if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName))
 934           continue;
 935         const uint8_t *FuncProfileAddr = Start + NameOffset.second;
 936         if (std::error_code EC = readFuncProfile(FuncProfileAddr))
 937           return EC;
 938       }
 939     } else {
 940       assert(!useFuncOffsetList());
 941       for (auto Name : FuncsToUse) {
 942         auto iter = FuncOffsetTable.find(MD5Hash(Name));
 943         if (iter == FuncOffsetTable.end())
 944           continue;
 945         const uint8_t *FuncProfileAddr = Start + iter->second;
 946         if (std::error_code EC = readFuncProfile(FuncProfileAddr))
 947           return EC;
 948       }
 949     }
 950     Data = End;
 951   }
 952   assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
 953          "Cannot have both context-sensitive and regular profile");
 954   assert((!CSProfileCount || ProfileIsCS) &&
 955          "Section flag should be consistent with actual profile");
 956   return sampleprof_error::success;
 957 }
 958
 959 std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {
 960   if (!ProfSymList)
 961     ProfSymList = std::make_unique<ProfileSymbolList>();
 962
 963   if (std::error_code EC = ProfSymList->read(Data, End - Data))
 964     return EC;
 965
 966   Data = End;
 967   return sampleprof_error::success;
 968 }
 969
 970 std::error_code SampleProfileReaderExtBinaryBase::decompressSection(
 971     const uint8_t *SecStart, const uint64_t SecSize,
 972     const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {
 973   Data = SecStart;
 974   End = SecStart + SecSize;
 975   auto DecompressSize = readNumber<uint64_t>();
 976   if (std::error_code EC = DecompressSize.getError())
 977     return EC;
 978   DecompressBufSize = *DecompressSize;
 979
 980   auto CompressSize = readNumber<uint64_t>();
 981   if (std::error_code EC = CompressSize.getError())
 982     return EC;
 983
 984   if (!llvm::compression::zlib::isAvailable())
 985     return sampleprof_error::zlib_unavailable;
 986
 987   uint8_t *Buffer = Allocator.Allocate<uint8_t>(DecompressBufSize);
 988   size_t UCSize = DecompressBufSize;
 989   llvm::Error E = compression::zlib::decompress(ArrayRef(Data, *CompressSize),
 990                                                 Buffer, UCSize);
 991   if (E)
 992     return sampleprof_error::uncompress_failed;
 993   DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);
 994   return sampleprof_error::success;
 995 }
 996
 997 std::error_code SampleProfileReaderExtBinaryBase::readImpl() {
 998   const uint8_t *BufStart =
 999       reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1000
1001   for (auto &Entry : SecHdrTable) {
1002     // Skip empty section.
1003     if (!Entry.Size)
1004       continue;
1005
1006     // Skip sections without context when SkipFlatProf is true.
1007     if (SkipFlatProf && hasSecFlag(Entry, SecCommonFlags::SecFlagFlat))
1008       continue;
1009
1010     const uint8_t *SecStart = BufStart + Entry.Offset;
1011     uint64_t SecSize = Entry.Size;
1012
1013     // If the section is compressed, decompress it into a buffer
1014     // DecompressBuf before reading the actual data. The pointee of
1015     // 'Data' will be changed to buffer hold by DecompressBuf
1016     // temporarily when reading the actual data.
1017     bool isCompressed = hasSecFlag(Entry, SecCommonFlags::SecFlagCompress);
1018     if (isCompressed) {
1019       const uint8_t *DecompressBuf;
1020       uint64_t DecompressBufSize;
1021       if (std::error_code EC = decompressSection(
1022               SecStart, SecSize, DecompressBuf, DecompressBufSize))
1023         return EC;
1024       SecStart = DecompressBuf;
1025       SecSize = DecompressBufSize;
1026     }
1027
1028     if (std::error_code EC = readOneSection(SecStart, SecSize, Entry))
1029       return EC;
1030     if (Data != SecStart + SecSize)
1031       return sampleprof_error::malformed;
1032
1033     // Change the pointee of 'Data' from DecompressBuf to original Buffer.
1034     if (isCompressed) {
1035       Data = BufStart + Entry.Offset;
1036       End = BufStart + Buffer->getBufferSize();
1037     }
1038   }
1039
1040   return sampleprof_error::success;
1041 }
1042
1043 std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {
1044   if (Magic == SPMagic())
1045     return sampleprof_error::success;
1046   return sampleprof_error::bad_magic;
1047 }
1048
1049 std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {
1050   if (Magic == SPMagic(SPF_Ext_Binary))
1051     return sampleprof_error::success;
1052   return sampleprof_error::bad_magic;
1053 }
1054
1055 std::error_code SampleProfileReaderBinary::readNameTable() {
1056   auto Size = readNumber<size_t>();
1057   if (std::error_code EC = Size.getError())
1058     return EC;
1059
1060   // Normally if useMD5 is true, the name table should have MD5 values, not
1061   // strings, however in the case that ExtBinary profile has multiple name
1062   // tables mixing string and MD5, all of them have to be normalized to use MD5,
1063   // because optimization passes can only handle either type.
1064   bool UseMD5 = useMD5();
1065   if (UseMD5)
1066     MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
1067
1068   NameTable.clear();
1069   NameTable.reserve(*Size);
1070   if (!ProfileIsCS) {
1071     MD5SampleContextTable.clear();
1072     if (UseMD5)
1073       MD5SampleContextTable.reserve(*Size);
1074     else
1075       // If we are using strings, delay MD5 computation since only a portion of
1076       // names are used by top level functions. Use 0 to indicate MD5 value is
1077       // to be calculated as no known string has a MD5 value of 0.
1078       MD5SampleContextTable.resize(*Size);
1079   }
1080   for (size_t I = 0; I < *Size; ++I) {
1081     auto Name(readString());
1082     if (std::error_code EC = Name.getError())
1083       return EC;
1084     if (UseMD5) {
1085       uint64_t FID = hashFuncName(*Name);
1086       if (!ProfileIsCS)
1087         MD5SampleContextTable.emplace_back(FID);
1088       NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID)));
1089     } else
1090       NameTable.push_back(*Name);
1091   }
1092   if (!ProfileIsCS)
1093     MD5SampleContextStart = MD5SampleContextTable.data();
1094   return sampleprof_error::success;
1095 }
1096
1097 std::error_code
1098 SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
1099                                                    bool FixedLengthMD5) {
1100   if (FixedLengthMD5) {
1101     if (!IsMD5)
1102       errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";
1103     auto Size = readNumber<size_t>();
1104     if (std::error_code EC = Size.getError())
1105       return EC;
1106
1107     assert(Data + (*Size) * sizeof(uint64_t) == End &&
1108            "Fixed length MD5 name table does not contain specified number of "
1109            "entries");
1110     if (Data + (*Size) * sizeof(uint64_t) > End)
1111       return sampleprof_error::truncated;
1112
1113     // Preallocate and initialize NameTable so we can check whether a name
1114     // index has been read before by checking whether the element in the
1115     // NameTable is empty, meanwhile readStringIndex can do the boundary
1116     // check using the size of NameTable.
1117     MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
1118     NameTable.clear();
1119     NameTable.resize(*Size);
1120     MD5NameMemStart = Data;
1121     if (!ProfileIsCS)
1122       MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
1123     Data = Data + (*Size) * sizeof(uint64_t);
1124     return sampleprof_error::success;
1125   }
1126
1127   if (IsMD5) {
1128     assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");
1129     auto Size = readNumber<size_t>();
1130     if (std::error_code EC = Size.getError())
1131       return EC;
1132
1133     MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
1134     NameTable.clear();
1135     NameTable.reserve(*Size);
1136     if (!ProfileIsCS)
1137       MD5SampleContextTable.resize(*Size);
1138     for (size_t I = 0; I < *Size; ++I) {
1139       auto FID = readNumber<uint64_t>();
1140       if (std::error_code EC = FID.getError())
1141         return EC;
1142       if (!ProfileIsCS)
1143         support::endian::write64le(&MD5SampleContextTable[I], *FID);
1144       NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID)));
1145     }
1146     if (!ProfileIsCS)
1147       MD5SampleContextStart = MD5SampleContextTable.data();
1148     return sampleprof_error::success;
1149   }
1150
1151   return SampleProfileReaderBinary::readNameTable();
1152 }
1153
1154 // Read in the CS name table section, which basically contains a list of context
1155 // vectors. Each element of a context vector, aka a frame, refers to the
1156 // underlying raw function names that are stored in the name table, as well as
1157 // a callsite identifier that only makes sense for non-leaf frames.
1158 std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {
1159   auto Size = readNumber<size_t>();
1160   if (std::error_code EC = Size.getError())
1161     return EC;
1162
1163   CSNameTable.clear();
1164   CSNameTable.reserve(*Size);
1165   if (ProfileIsCS) {
1166     // Delay MD5 computation of CS context until they are needed. Use 0 to
1167     // indicate MD5 value is to be calculated as no known string has a MD5
1168     // value of 0.
1169     MD5SampleContextTable.clear();
1170     MD5SampleContextTable.resize(*Size);
1171     MD5SampleContextStart = MD5SampleContextTable.data();
1172   }
1173   for (size_t I = 0; I < *Size; ++I) {
1174     CSNameTable.emplace_back(SampleContextFrameVector());
1175     auto ContextSize = readNumber<uint32_t>();
1176     if (std::error_code EC = ContextSize.getError())
1177       return EC;
1178     for (uint32_t J = 0; J < *ContextSize; ++J) {
1179       auto FName(readStringFromTable());
1180       if (std::error_code EC = FName.getError())
1181         return EC;
1182       auto LineOffset = readNumber<uint64_t>();
1183       if (std::error_code EC = LineOffset.getError())
1184         return EC;
1185
1186       if (!isOffsetLegal(*LineOffset))
1187         return std::error_code();
1188
1189       auto Discriminator = readNumber<uint64_t>();
1190       if (std::error_code EC = Discriminator.getError())
1191         return EC;
1192
1193       CSNameTable.back().emplace_back(
1194           FName.get(), LineLocation(LineOffset.get(), Discriminator.get()));
1195     }
1196   }
1197
1198   return sampleprof_error::success;
1199 }
1200
1201 std::error_code
1202 SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1203                                                    FunctionSamples *FProfile) {
1204   if (Data < End) {
1205     if (ProfileIsProbeBased) {
1206       auto Checksum = readNumber<uint64_t>();
1207       if (std::error_code EC = Checksum.getError())
1208         return EC;
1209       if (FProfile)
1210         FProfile->setFunctionHash(*Checksum);
1211     }
1212
1213     if (ProfileHasAttribute) {
1214       auto Attributes = readNumber<uint32_t>();
1215       if (std::error_code EC = Attributes.getError())
1216         return EC;
1217       if (FProfile)
1218         FProfile->getContext().setAllAttributes(*Attributes);
1219     }
1220
1221     if (!ProfileIsCS) {
1222       // Read all the attributes for inlined function calls.
1223       auto NumCallsites = readNumber<uint32_t>();
1224       if (std::error_code EC = NumCallsites.getError())
1225         return EC;
1226
1227       for (uint32_t J = 0; J < *NumCallsites; ++J) {
1228         auto LineOffset = readNumber<uint64_t>();
1229         if (std::error_code EC = LineOffset.getError())
1230           return EC;
1231
1232         auto Discriminator = readNumber<uint64_t>();
1233         if (std::error_code EC = Discriminator.getError())
1234           return EC;
1235
1236         auto FContextHash(readSampleContextFromTable());
1237         if (std::error_code EC = FContextHash.getError())
1238           return EC;
1239
1240         auto &[FContext, Hash] = *FContextHash;
1241         FunctionSamples *CalleeProfile = nullptr;
1242         if (FProfile) {
1243           CalleeProfile = const_cast<FunctionSamples *>(
1244               &FProfile->functionSamplesAt(LineLocation(
1245                   *LineOffset,
1246                   *Discriminator))[std::string(FContext.getName())]);
1247         }
1248         if (std::error_code EC =
1249                 readFuncMetadata(ProfileHasAttribute, CalleeProfile))
1250           return EC;
1251       }
1252     }
1253   }
1254
1255   return sampleprof_error::success;
1256 }
1257
1258 std::error_code
1259 SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
1260   while (Data < End) {
1261     auto FContextHash(readSampleContextFromTable());
1262     if (std::error_code EC = FContextHash.getError())
1263       return EC;
1264     auto &[FContext, Hash] = *FContextHash;
1265     FunctionSamples *FProfile = nullptr;
1266     auto It = Profiles.find(FContext);
1267     if (It != Profiles.end())
1268       FProfile = &It->second;
1269
1270     if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1271       return EC;
1272   }
1273
1274   assert(Data == End && "More data is read than expected");
1275   return sampleprof_error::success;
1276 }
1277
1278 std::error_code
1279 SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {
1280   SecHdrTableEntry Entry;
1281   auto Type = readUnencodedNumber<uint64_t>();
1282   if (std::error_code EC = Type.getError())
1283     return EC;
1284   Entry.Type = static_cast<SecType>(*Type);
1285
1286   auto Flags = readUnencodedNumber<uint64_t>();
1287   if (std::error_code EC = Flags.getError())
1288     return EC;
1289   Entry.Flags = *Flags;
1290
1291   auto Offset = readUnencodedNumber<uint64_t>();
1292   if (std::error_code EC = Offset.getError())
1293     return EC;
1294   Entry.Offset = *Offset;
1295
1296   auto Size = readUnencodedNumber<uint64_t>();
1297   if (std::error_code EC = Size.getError())
1298     return EC;
1299   Entry.Size = *Size;
1300
1301   Entry.LayoutIndex = Idx;
1302   SecHdrTable.push_back(std::move(Entry));
1303   return sampleprof_error::success;
1304 }
1305
1306 std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {
1307   auto EntryNum = readUnencodedNumber<uint64_t>();
1308   if (std::error_code EC = EntryNum.getError())
1309     return EC;
1310
1311   for (uint64_t i = 0; i < (*EntryNum); i++)
1312     if (std::error_code EC = readSecHdrTableEntry(i))
1313       return EC;
1314
1315   return sampleprof_error::success;
1316 }
1317
1318 std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
1319   const uint8_t *BufStart =
1320       reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1321   Data = BufStart;
1322   End = BufStart + Buffer->getBufferSize();
1323
1324   if (std::error_code EC = readMagicIdent())
1325     return EC;
1326
1327   if (std::error_code EC = readSecHdrTable())
1328     return EC;
1329
1330   return sampleprof_error::success;
1331 }
1332
1333 uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
1334   uint64_t Size = 0;
1335   for (auto &Entry : SecHdrTable) {
1336     if (Entry.Type == Type)
1337       Size += Entry.Size;
1338   }
1339   return Size;
1340 }
1341
1342 uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
1343   // Sections in SecHdrTable is not necessarily in the same order as
1344   // sections in the profile because section like FuncOffsetTable needs
1345   // to be written after section LBRProfile but needs to be read before
1346   // section LBRProfile, so we cannot simply use the last entry in
1347   // SecHdrTable to calculate the file size.
1348   uint64_t FileSize = 0;
1349   for (auto &Entry : SecHdrTable) {
1350     FileSize = std::max(Entry.Offset + Entry.Size, FileSize);
1351   }
1352   return FileSize;
1353 }
1354
1355 static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
1356   std::string Flags;
1357   if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))
1358     Flags.append("{compressed,");
1359   else
1360     Flags.append("{");
1361
1362   if (hasSecFlag(Entry, SecCommonFlags::SecFlagFlat))
1363     Flags.append("flat,");
1364
1365   switch (Entry.Type) {
1366   case SecNameTable:
1367     if (hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5))
1368       Flags.append("fixlenmd5,");
1369     else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))
1370       Flags.append("md5,");
1371     if (hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix))
1372       Flags.append("uniq,");
1373     break;
1374   case SecProfSummary:
1375     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
1376       Flags.append("partial,");
1377     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext))
1378       Flags.append("context,");
1379     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined))
1380       Flags.append("preInlined,");
1381     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))
1382       Flags.append("fs-discriminator,");
1383     break;
1384   case SecFuncOffsetTable:
1385     if (hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered))
1386       Flags.append("ordered,");
1387     break;
1388   case SecFuncMetadata:
1389     if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased))
1390       Flags.append("probe,");
1391     if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute))
1392       Flags.append("attr,");
1393     break;
1394   default:
1395     break;
1396   }
1397   char &last = Flags.back();
1398   if (last == ',')
1399     last = '}';
1400   else
1401     Flags.append("}");
1402   return Flags;
1403 }
1404
1405 bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
1406   uint64_t TotalSecsSize = 0;
1407   for (auto &Entry : SecHdrTable) {
1408     OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
1409        << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)
1410        << "\n";
1411     ;
1412     TotalSecsSize += Entry.Size;
1413   }
1414   uint64_t HeaderSize = SecHdrTable.front().Offset;
1415   assert(HeaderSize + TotalSecsSize == getFileSize() &&
1416          "Size of 'header + sections' doesn't match the total size of profile");
1417
1418   OS << "Header Size: " << HeaderSize << "\n";
1419   OS << "Total Sections Size: " << TotalSecsSize << "\n";
1420   OS << "File Size: " << getFileSize() << "\n";
1421   return true;
1422 }
1423
1424 std::error_code SampleProfileReaderBinary::readMagicIdent() {
1425   // Read and check the magic identifier.
1426   auto Magic = readNumber<uint64_t>();
1427   if (std::error_code EC = Magic.getError())
1428     return EC;
1429   else if (std::error_code EC = verifySPMagic(*Magic))
1430     return EC;
1431
1432   // Read the version number.
1433   auto Version = readNumber<uint64_t>();
1434   if (std::error_code EC = Version.getError())
1435     return EC;
1436   else if (*Version != SPVersion())
1437     return sampleprof_error::unsupported_version;
1438
1439   return sampleprof_error::success;
1440 }
1441
1442 std::error_code SampleProfileReaderBinary::readHeader() {
1443   Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());
1444   End = Data + Buffer->getBufferSize();
1445
1446   if (std::error_code EC = readMagicIdent())
1447     return EC;
1448
1449   if (std::error_code EC = readSummary())
1450     return EC;
1451
1452   if (std::error_code EC = readNameTable())
1453     return EC;
1454   return sampleprof_error::success;
1455 }
1456
1457 std::error_code SampleProfileReaderBinary::readSummaryEntry(
1458     std::vector<ProfileSummaryEntry> &Entries) {
1459   auto Cutoff = readNumber<uint64_t>();
1460   if (std::error_code EC = Cutoff.getError())
1461     return EC;
1462
1463   auto MinBlockCount = readNumber<uint64_t>();
1464   if (std::error_code EC = MinBlockCount.getError())
1465     return EC;
1466
1467   auto NumBlocks = readNumber<uint64_t>();
1468   if (std::error_code EC = NumBlocks.getError())
1469     return EC;
1470
1471   Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);
1472   return sampleprof_error::success;
1473 }
1474
1475 std::error_code SampleProfileReaderBinary::readSummary() {
1476   auto TotalCount = readNumber<uint64_t>();
1477   if (std::error_code EC = TotalCount.getError())
1478     return EC;
1479
1480   auto MaxBlockCount = readNumber<uint64_t>();
1481   if (std::error_code EC = MaxBlockCount.getError())
1482     return EC;
1483
1484   auto MaxFunctionCount = readNumber<uint64_t>();
1485   if (std::error_code EC = MaxFunctionCount.getError())
1486     return EC;
1487
1488   auto NumBlocks = readNumber<uint64_t>();
1489   if (std::error_code EC = NumBlocks.getError())
1490     return EC;
1491
1492   auto NumFunctions = readNumber<uint64_t>();
1493   if (std::error_code EC = NumFunctions.getError())
1494     return EC;
1495
1496   auto NumSummaryEntries = readNumber<uint64_t>();
1497   if (std::error_code EC = NumSummaryEntries.getError())
1498     return EC;
1499
1500   std::vector<ProfileSummaryEntry> Entries;
1501   for (unsigned i = 0; i < *NumSummaryEntries; i++) {
1502     std::error_code EC = readSummaryEntry(Entries);
1503     if (EC != sampleprof_error::success)
1504       return EC;
1505   }
1506   Summary = std::make_unique<ProfileSummary>(
1507       ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,
1508       *MaxFunctionCount, *NumBlocks, *NumFunctions);
1509
1510   return sampleprof_error::success;
1511 }
1512
1513 bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {
1514   const uint8_t *Data =
1515       reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1516   uint64_t Magic = decodeULEB128(Data);
1517   return Magic == SPMagic();
1518 }
1519
1520 bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {
1521   const uint8_t *Data =
1522       reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());
1523   uint64_t Magic = decodeULEB128(Data);
1524   return Magic == SPMagic(SPF_Ext_Binary);
1525 }
1526
1527 std::error_code SampleProfileReaderGCC::skipNextWord() {
1528   uint32_t dummy;
1529   if (!GcovBuffer.readInt(dummy))
1530     return sampleprof_error::truncated;
1531   return sampleprof_error::success;
1532 }
1533
1534 template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {
1535   if (sizeof(T) <= sizeof(uint32_t)) {
1536     uint32_t Val;
1537     if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())
1538       return static_cast<T>(Val);
1539   } else if (sizeof(T) <= sizeof(uint64_t)) {
1540     uint64_t Val;
1541     if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())
1542       return static_cast<T>(Val);
1543   }
1544
1545   std::error_code EC = sampleprof_error::malformed;
1546   reportError(0, EC.message());
1547   return EC;
1548 }
1549
1550 ErrorOr<StringRef> SampleProfileReaderGCC::readString() {
1551   StringRef Str;
1552   if (!GcovBuffer.readString(Str))
1553     return sampleprof_error::truncated;
1554   return Str;
1555 }
1556
1557 std::error_code SampleProfileReaderGCC::readHeader() {
1558   // Read the magic identifier.
1559   if (!GcovBuffer.readGCDAFormat())
1560     return sampleprof_error::unrecognized_format;
1561
1562   // Read the version number. Note - the GCC reader does not validate this
1563   // version, but the profile creator generates v704.
1564   GCOV::GCOVVersion version;
1565   if (!GcovBuffer.readGCOVVersion(version))
1566     return sampleprof_error::unrecognized_format;
1567
1568   if (version != GCOV::V407)
1569     return sampleprof_error::unsupported_version;
1570
1571   // Skip the empty integer.
1572   if (std::error_code EC = skipNextWord())
1573     return EC;
1574
1575   return sampleprof_error::success;
1576 }
1577
1578 std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {
1579   uint32_t Tag;
1580   if (!GcovBuffer.readInt(Tag))
1581     return sampleprof_error::truncated;
1582
1583   if (Tag != Expected)
1584     return sampleprof_error::malformed;
1585
1586   if (std::error_code EC = skipNextWord())
1587     return EC;
1588
1589   return sampleprof_error::success;
1590 }
1591
1592 std::error_code SampleProfileReaderGCC::readNameTable() {
1593   if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))
1594     return EC;
1595
1596   uint32_t Size;
1597   if (!GcovBuffer.readInt(Size))
1598     return sampleprof_error::truncated;
1599
1600   for (uint32_t I = 0; I < Size; ++I) {
1601     StringRef Str;
1602     if (!GcovBuffer.readString(Str))
1603       return sampleprof_error::truncated;
1604     Names.push_back(std::string(Str));
1605   }
1606
1607   return sampleprof_error::success;
1608 }
1609
1610 std::error_code SampleProfileReaderGCC::readFunctionProfiles() {
1611   if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))
1612     return EC;
1613
1614   uint32_t NumFunctions;
1615   if (!GcovBuffer.readInt(NumFunctions))
1616     return sampleprof_error::truncated;
1617
1618   InlineCallStack Stack;
1619   for (uint32_t I = 0; I < NumFunctions; ++I)
1620     if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))
1621       return EC;
1622
1623   computeSummary();
1624   return sampleprof_error::success;
1625 }
1626
1627 std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
1628     const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {
1629   uint64_t HeadCount = 0;
1630   if (InlineStack.size() == 0)
1631     if (!GcovBuffer.readInt64(HeadCount))
1632       return sampleprof_error::truncated;
1633
1634   uint32_t NameIdx;
1635   if (!GcovBuffer.readInt(NameIdx))
1636     return sampleprof_error::truncated;
1637
1638   StringRef Name(Names[NameIdx]);
1639
1640   uint32_t NumPosCounts;
1641   if (!GcovBuffer.readInt(NumPosCounts))
1642     return sampleprof_error::truncated;
1643
1644   uint32_t NumCallsites;
1645   if (!GcovBuffer.readInt(NumCallsites))
1646     return sampleprof_error::truncated;
1647
1648   FunctionSamples *FProfile = nullptr;
1649   if (InlineStack.size() == 0) {
1650     // If this is a top function that we have already processed, do not
1651     // update its profile again.  This happens in the presence of
1652     // function aliases.  Since these aliases share the same function
1653     // body, there will be identical replicated profiles for the
1654     // original function.  In this case, we simply not bother updating
1655     // the profile of the original function.
1656     FProfile = &Profiles[Name];
1657     FProfile->addHeadSamples(HeadCount);
1658     if (FProfile->getTotalSamples() > 0)
1659       Update = false;
1660   } else {
1661     // Otherwise, we are reading an inlined instance. The top of the
1662     // inline stack contains the profile of the caller. Insert this
1663     // callee in the caller's CallsiteMap.
1664     FunctionSamples *CallerProfile = InlineStack.front();
1665     uint32_t LineOffset = Offset >> 16;
1666     uint32_t Discriminator = Offset & 0xffff;
1667     FProfile = &CallerProfile->functionSamplesAt(
1668         LineLocation(LineOffset, Discriminator))[std::string(Name)];
1669   }
1670   FProfile->setName(Name);
1671
1672   for (uint32_t I = 0; I < NumPosCounts; ++I) {
1673     uint32_t Offset;
1674     if (!GcovBuffer.readInt(Offset))
1675       return sampleprof_error::truncated;
1676
1677     uint32_t NumTargets;
1678     if (!GcovBuffer.readInt(NumTargets))
1679       return sampleprof_error::truncated;
1680
1681     uint64_t Count;
1682     if (!GcovBuffer.readInt64(Count))
1683       return sampleprof_error::truncated;
1684
1685     // The line location is encoded in the offset as:
1686     //   high 16 bits: line offset to the start of the function.
1687     //   low 16 bits: discriminator.
1688     uint32_t LineOffset = Offset >> 16;
1689     uint32_t Discriminator = Offset & 0xffff;
1690
1691     InlineCallStack NewStack;
1692     NewStack.push_back(FProfile);
1693     llvm::append_range(NewStack, InlineStack);
1694     if (Update) {
1695       // Walk up the inline stack, adding the samples on this line to
1696       // the total sample count of the callers in the chain.
1697       for (auto *CallerProfile : NewStack)
1698         CallerProfile->addTotalSamples(Count);
1699
1700       // Update the body samples for the current profile.
1701       FProfile->addBodySamples(LineOffset, Discriminator, Count);
1702     }
1703
1704     // Process the list of functions called at an indirect call site.
1705     // These are all the targets that a function pointer (or virtual
1706     // function) resolved at runtime.
1707     for (uint32_t J = 0; J < NumTargets; J++) {
1708       uint32_t HistVal;
1709       if (!GcovBuffer.readInt(HistVal))
1710         return sampleprof_error::truncated;
1711
1712       if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)
1713         return sampleprof_error::malformed;
1714
1715       uint64_t TargetIdx;
1716       if (!GcovBuffer.readInt64(TargetIdx))
1717         return sampleprof_error::truncated;
1718       StringRef TargetName(Names[TargetIdx]);
1719
1720       uint64_t TargetCount;
1721       if (!GcovBuffer.readInt64(TargetCount))
1722         return sampleprof_error::truncated;
1723
1724       if (Update)
1725         FProfile->addCalledTargetSamples(LineOffset, Discriminator,
1726                                          TargetName, TargetCount);
1727     }
1728   }
1729
1730   // Process all the inlined callers into the current function. These
1731   // are all the callsites that were inlined into this function.
1732   for (uint32_t I = 0; I < NumCallsites; I++) {
1733     // The offset is encoded as:
1734     //   high 16 bits: line offset to the start of the function.
1735     //   low 16 bits: discriminator.
1736     uint32_t Offset;
1737     if (!GcovBuffer.readInt(Offset))
1738       return sampleprof_error::truncated;
1739     InlineCallStack NewStack;
1740     NewStack.push_back(FProfile);
1741     llvm::append_range(NewStack, InlineStack);
1742     if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))
1743       return EC;
1744   }
1745
1746   return sampleprof_error::success;
1747 }
1748
1749 /// Read a GCC AutoFDO profile.
1750 ///
1751 /// This format is generated by the Linux Perf conversion tool at
1752 /// https://github.com/google/autofdo.
1753 std::error_code SampleProfileReaderGCC::readImpl() {
1754   assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");
1755   // Read the string table.
1756   if (std::error_code EC = readNameTable())
1757     return EC;
1758
1759   // Read the source profile.
1760   if (std::error_code EC = readFunctionProfiles())
1761     return EC;
1762
1763   return sampleprof_error::success;
1764 }
1765
1766 bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {
1767   StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart()));
1768   return Magic == "adcg*704";
1769 }
1770
1771 void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
1772   // If the reader uses MD5 to represent string, we can't remap it because
1773   // we don't know what the original function names were.
1774   if (Reader.useMD5()) {
1775     Ctx.diagnose(DiagnosticInfoSampleProfile(
1776         Reader.getBuffer()->getBufferIdentifier(),
1777         "Profile data remapping cannot be applied to profile data "
1778         "using MD5 names (original mangled names are not available).",
1779         DS_Warning));
1780     return;
1781   }
1782
1783   // CSSPGO-TODO: Remapper is not yet supported.
1784   // We will need to remap the entire context string.
1785   assert(Remappings && "should be initialized while creating remapper");
1786   for (auto &Sample : Reader.getProfiles()) {
1787     DenseSet<StringRef> NamesInSample;
1788     Sample.second.findAllNames(NamesInSample);
1789     for (auto &Name : NamesInSample)
1790       if (auto Key = Remappings->insert(Name))
1791         NameMap.insert({Key, Name});
1792   }
1793
1794   RemappingApplied = true;
1795 }
1796
1797 std::optional<StringRef>
1798 SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {
1799   if (auto Key = Remappings->lookup(Fname))
1800     return NameMap.lookup(Key);
1801   return std::nullopt;
1802 }
1803
1804 /// Prepare a memory buffer for the contents of \p Filename.
1805 ///
1806 /// \returns an error code indicating the status of the buffer.
1807 static ErrorOr<std::unique_ptr<MemoryBuffer>>
1808 setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
1809   auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
1810                                            : FS.getBufferForFile(Filename);
1811   if (std::error_code EC = BufferOrErr.getError())
1812     return EC;
1813   auto Buffer = std::move(BufferOrErr.get());
1814
1815   return std::move(Buffer);
1816 }
1817
1818 /// Create a sample profile reader based on the format of the input file.
1819 ///
1820 /// \param Filename The file to open.
1821 ///
1822 /// \param C The LLVM context to use to emit diagnostics.
1823 ///
1824 /// \param P The FSDiscriminatorPass.
1825 ///
1826 /// \param RemapFilename The file used for profile remapping.
1827 ///
1828 /// \returns an error code indicating the status of the created reader.
1829 ErrorOr<std::unique_ptr<SampleProfileReader>>
1830 SampleProfileReader::create(const std::string Filename, LLVMContext &C,
1831                             vfs::FileSystem &FS, FSDiscriminatorPass P,
1832                             const std::string RemapFilename) {
1833   auto BufferOrError = setupMemoryBuffer(Filename, FS);
1834   if (std::error_code EC = BufferOrError.getError())
1835     return EC;
1836   return create(BufferOrError.get(), C, FS, P, RemapFilename);
1837 }
1838
1839 /// Create a sample profile remapper from the given input, to remap the
1840 /// function names in the given profile data.
1841 ///
1842 /// \param Filename The file to open.
1843 ///
1844 /// \param Reader The profile reader the remapper is going to be applied to.
1845 ///
1846 /// \param C The LLVM context to use to emit diagnostics.
1847 ///
1848 /// \returns an error code indicating the status of the created reader.
1849 ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1850 SampleProfileReaderItaniumRemapper::create(const std::string Filename,
1851                                            vfs::FileSystem &FS,
1852                                            SampleProfileReader &Reader,
1853                                            LLVMContext &C) {
1854   auto BufferOrError = setupMemoryBuffer(Filename, FS);
1855   if (std::error_code EC = BufferOrError.getError())
1856     return EC;
1857   return create(BufferOrError.get(), Reader, C);
1858 }
1859
1860 /// Create a sample profile remapper from the given input, to remap the
1861 /// function names in the given profile data.
1862 ///
1863 /// \param B The memory buffer to create the reader from (assumes ownership).
1864 ///
1865 /// \param C The LLVM context to use to emit diagnostics.
1866 ///
1867 /// \param Reader The profile reader the remapper is going to be applied to.
1868 ///
1869 /// \returns an error code indicating the status of the created reader.
1870 ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>
1871 SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,
1872                                            SampleProfileReader &Reader,
1873                                            LLVMContext &C) {
1874   auto Remappings = std::make_unique<SymbolRemappingReader>();
1875   if (Error E = Remappings->read(*B)) {
1876     handleAllErrors(
1877         std::move(E), [&](const SymbolRemappingParseError &ParseError) {
1878           C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(),
1879                                                  ParseError.getLineNum(),
1880                                                  ParseError.getMessage()));
1881         });
1882     return sampleprof_error::malformed;
1883   }
1884
1885   return std::make_unique<SampleProfileReaderItaniumRemapper>(
1886       std::move(B), std::move(Remappings), Reader);
1887 }
1888
1889 /// Create a sample profile reader based on the format of the input data.
1890 ///
1891 /// \param B The memory buffer to create the reader from (assumes ownership).
1892 ///
1893 /// \param C The LLVM context to use to emit diagnostics.
1894 ///
1895 /// \param P The FSDiscriminatorPass.
1896 ///
1897 /// \param RemapFilename The file used for profile remapping.
1898 ///
1899 /// \returns an error code indicating the status of the created reader.
1900 ErrorOr<std::unique_ptr<SampleProfileReader>>
1901 SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,
1902                             vfs::FileSystem &FS, FSDiscriminatorPass P,
1903                             const std::string RemapFilename) {
1904   std::unique_ptr<SampleProfileReader> Reader;
1905   if (SampleProfileReaderRawBinary::hasFormat(*B))
1906     Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));
1907   else if (SampleProfileReaderExtBinary::hasFormat(*B))
1908     Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));
1909   else if (SampleProfileReaderGCC::hasFormat(*B))
1910     Reader.reset(new SampleProfileReaderGCC(std::move(B), C));
1911   else if (SampleProfileReaderText::hasFormat(*B))
1912     Reader.reset(new SampleProfileReaderText(std::move(B), C));
1913   else
1914     return sampleprof_error::unrecognized_format;
1915
1916   if (!RemapFilename.empty()) {
1917     auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(
1918         RemapFilename, FS, *Reader, C);
1919     if (std::error_code EC = ReaderOrErr.getError()) {
1920       std::string Msg = "Could not create remapper: " + EC.message();
1921       C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg));
1922       return EC;
1923     }
1924     Reader->Remapper = std::move(ReaderOrErr.get());
1925   }
1926
1927   if (std::error_code EC = Reader->readHeader()) {
1928     return EC;
1929   }
1930
1931   Reader->setDiscriminatorMaskedBitFrom(P);
1932
1933   return std::move(Reader);
1934 }
1935
1936 // For text and GCC file formats, we compute the summary after reading the
1937 // profile. Binary format has the profile summary in its header.
1938 void SampleProfileReader::computeSummary() {
1939   SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
1940   Summary = Builder.computeSummaryForProfiles(Profiles);
1941 }