llvm/lib/ProfileData/InstrProf.cpp

   1 //===- InstrProf.cpp - Instrumented profiling format support --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains support for clang's instrumentation based PGO and
  10 // coverage.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/ProfileData/InstrProf.h"
  15 #include "llvm/ADT/ArrayRef.h"
  16 #include "llvm/ADT/SmallVector.h"
  17 #include "llvm/ADT/StringExtras.h"
  18 #include "llvm/ADT/StringRef.h"
  19 #include "llvm/Config/config.h"
  20 #include "llvm/IR/Constant.h"
  21 #include "llvm/IR/Constants.h"
  22 #include "llvm/IR/Function.h"
  23 #include "llvm/IR/GlobalValue.h"
  24 #include "llvm/IR/GlobalVariable.h"
  25 #include "llvm/IR/Instruction.h"
  26 #include "llvm/IR/LLVMContext.h"
  27 #include "llvm/IR/MDBuilder.h"
  28 #include "llvm/IR/Metadata.h"
  29 #include "llvm/IR/Module.h"
  30 #include "llvm/IR/Type.h"
  31 #include "llvm/ProfileData/InstrProfReader.h"
  32 #include "llvm/Support/Casting.h"
  33 #include "llvm/Support/CommandLine.h"
  34 #include "llvm/Support/Compiler.h"
  35 #include "llvm/Support/Compression.h"
  36 #include "llvm/Support/Debug.h"
  37 #include "llvm/Support/Endian.h"
  38 #include "llvm/Support/Error.h"
  39 #include "llvm/Support/ErrorHandling.h"
  40 #include "llvm/Support/LEB128.h"
  41 #include "llvm/Support/MathExtras.h"
  42 #include "llvm/Support/Path.h"
  43 #include "llvm/Support/SwapByteOrder.h"
  44 #include "llvm/Support/VirtualFileSystem.h"
  45 #include "llvm/TargetParser/Triple.h"
  46 #include <algorithm>
  47 #include <cassert>
  48 #include <cstddef>
  49 #include <cstdint>
  50 #include <cstring>
  51 #include <memory>
  52 #include <string>
  53 #include <system_error>
  54 #include <type_traits>
  55 #include <utility>
  56 #include <vector>
  57
  58 using namespace llvm;
  59
  60 #define DEBUG_TYPE "instrprof"
  61
  62 static cl::opt<bool> StaticFuncFullModulePrefix(
  63     "static-func-full-module-prefix", cl::init(true), cl::Hidden,
  64     cl::desc("Use full module build paths in the profile counter names for "
  65              "static functions."));
  66
  67 // This option is tailored to users that have different top-level directory in
  68 // profile-gen and profile-use compilation. Users need to specific the number
  69 // of levels to strip. A value larger than the number of directories in the
  70 // source file will strip all the directory names and only leave the basename.
  71 //
  72 // Note current ThinLTO module importing for the indirect-calls assumes
  73 // the source directory name not being stripped. A non-zero option value here
  74 // can potentially prevent some inter-module indirect-call-promotions.
  75 static cl::opt<unsigned> StaticFuncStripDirNamePrefix(
  76     "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden,
  77     cl::desc("Strip specified level of directory name from source path in "
  78              "the profile counter name for static functions."));
  79
  80 static std::string getInstrProfErrString(instrprof_error Err,
  81                                          const std::string &ErrMsg = "") {
  82   std::string Msg;
  83   raw_string_ostream OS(Msg);
  84
  85   switch (Err) {
  86   case instrprof_error::success:
  87     OS << "success";
  88     break;
  89   case instrprof_error::eof:
  90     OS << "end of File";
  91     break;
  92   case instrprof_error::unrecognized_format:
  93     OS << "unrecognized instrumentation profile encoding format";
  94     break;
  95   case instrprof_error::bad_magic:
  96     OS << "invalid instrumentation profile data (bad magic)";
  97     break;
  98   case instrprof_error::bad_header:
  99     OS << "invalid instrumentation profile data (file header is corrupt)";
 100     break;
 101   case instrprof_error::unsupported_version:
 102     OS << "unsupported instrumentation profile format version";
 103     break;
 104   case instrprof_error::unsupported_hash_type:
 105     OS << "unsupported instrumentation profile hash type";
 106     break;
 107   case instrprof_error::too_large:
 108     OS << "too much profile data";
 109     break;
 110   case instrprof_error::truncated:
 111     OS << "truncated profile data";
 112     break;
 113   case instrprof_error::malformed:
 114     OS << "malformed instrumentation profile data";
 115     break;
 116   case instrprof_error::missing_correlation_info:
 117     OS << "debug info/binary for correlation is required";
 118     break;
 119   case instrprof_error::unexpected_correlation_info:
 120     OS << "debug info/binary for correlation is not necessary";
 121     break;
 122   case instrprof_error::unable_to_correlate_profile:
 123     OS << "unable to correlate profile";
 124     break;
 125   case instrprof_error::invalid_prof:
 126     OS << "invalid profile created. Please file a bug "
 127           "at: " BUG_REPORT_URL
 128           " and include the profraw files that caused this error.";
 129     break;
 130   case instrprof_error::unknown_function:
 131     OS << "no profile data available for function";
 132     break;
 133   case instrprof_error::hash_mismatch:
 134     OS << "function control flow change detected (hash mismatch)";
 135     break;
 136   case instrprof_error::count_mismatch:
 137     OS << "function basic block count change detected (counter mismatch)";
 138     break;
 139   case instrprof_error::bitmap_mismatch:
 140     OS << "function bitmap size change detected (bitmap size mismatch)";
 141     break;
 142   case instrprof_error::counter_overflow:
 143     OS << "counter overflow";
 144     break;
 145   case instrprof_error::value_site_count_mismatch:
 146     OS << "function value site count change detected (counter mismatch)";
 147     break;
 148   case instrprof_error::compress_failed:
 149     OS << "failed to compress data (zlib)";
 150     break;
 151   case instrprof_error::uncompress_failed:
 152     OS << "failed to uncompress data (zlib)";
 153     break;
 154   case instrprof_error::empty_raw_profile:
 155     OS << "empty raw profile file";
 156     break;
 157   case instrprof_error::zlib_unavailable:
 158     OS << "profile uses zlib compression but the profile reader was built "
 159           "without zlib support";
 160     break;
 161   case instrprof_error::raw_profile_version_mismatch:
 162     OS << "raw profile version mismatch";
 163     break;
 164   case instrprof_error::counter_value_too_large:
 165     OS << "excessively large counter value suggests corrupted profile data";
 166     break;
 167   }
 168
 169   // If optional error message is not empty, append it to the message.
 170   if (!ErrMsg.empty())
 171     OS << ": " << ErrMsg;
 172
 173   return OS.str();
 174 }
 175
 176 namespace {
 177
 178 // FIXME: This class is only here to support the transition to llvm::Error. It
 179 // will be removed once this transition is complete. Clients should prefer to
 180 // deal with the Error value directly, rather than converting to error_code.
 181 class InstrProfErrorCategoryType : public std::error_category {
 182   const char *name() const noexcept override { return "llvm.instrprof"; }
 183
 184   std::string message(int IE) const override {
 185     return getInstrProfErrString(static_cast<instrprof_error>(IE));
 186   }
 187 };
 188
 189 } // end anonymous namespace
 190
 191 const std::error_category &llvm::instrprof_category() {
 192   static InstrProfErrorCategoryType ErrorCategory;
 193   return ErrorCategory;
 194 }
 195
 196 namespace {
 197
 198 const char *InstrProfSectNameCommon[] = {
 199 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)      \
 200   SectNameCommon,
 201 #include "llvm/ProfileData/InstrProfData.inc"
 202 };
 203
 204 const char *InstrProfSectNameCoff[] = {
 205 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)      \
 206   SectNameCoff,
 207 #include "llvm/ProfileData/InstrProfData.inc"
 208 };
 209
 210 const char *InstrProfSectNamePrefix[] = {
 211 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix)      \
 212   Prefix,
 213 #include "llvm/ProfileData/InstrProfData.inc"
 214 };
 215
 216 } // namespace
 217
 218 namespace llvm {
 219
 220 cl::opt<bool> DoInstrProfNameCompression(
 221     "enable-name-compression",
 222     cl::desc("Enable name/filename string compression"), cl::init(true));
 223
 224 cl::opt<bool> EnableVTableValueProfiling(
 225     "enable-vtable-value-profiling", cl::init(false),
 226     cl::desc("If true, the virtual table address will be instrumented to know "
 227              "the types of a C++ pointer. The information is used in indirect "
 228              "call promotion to do selective vtable-based comparison."));
 229
 230 cl::opt<bool> EnableVTableProfileUse(
 231     "enable-vtable-profile-use", cl::init(false),
 232     cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable "
 233              "profiles will be used by ICP pass for more efficient indirect "
 234              "call sequence. If false, type profiles won't be used."));
 235
 236 std::string getInstrProfSectionName(InstrProfSectKind IPSK,
 237                                     Triple::ObjectFormatType OF,
 238                                     bool AddSegmentInfo) {
 239   std::string SectName;
 240
 241   if (OF == Triple::MachO && AddSegmentInfo)
 242     SectName = InstrProfSectNamePrefix[IPSK];
 243
 244   if (OF == Triple::COFF)
 245     SectName += InstrProfSectNameCoff[IPSK];
 246   else
 247     SectName += InstrProfSectNameCommon[IPSK];
 248
 249   if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)
 250     SectName += ",regular,live_support";
 251
 252   return SectName;
 253 }
 254
 255 std::string InstrProfError::message() const {
 256   return getInstrProfErrString(Err, Msg);
 257 }
 258
 259 char InstrProfError::ID = 0;
 260
 261 std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage,
 262                            StringRef FileName,
 263                            uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
 264   // Value names may be prefixed with a binary '1' to indicate
 265   // that the backend should not modify the symbols due to any platform
 266   // naming convention. Do not include that '1' in the PGO profile name.
 267   if (Name[0] == '\1')
 268     Name = Name.substr(1);
 269
 270   std::string NewName = std::string(Name);
 271   if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
 272     // For local symbols, prepend the main file name to distinguish them.
 273     // Do not include the full path in the file name since there's no guarantee
 274     // that it will stay the same, e.g., if the files are checked out from
 275     // version control in different locations.
 276     if (FileName.empty())
 277       NewName = NewName.insert(0, "<unknown>:");
 278     else
 279       NewName = NewName.insert(0, FileName.str() + ":");
 280   }
 281   return NewName;
 282 }
 283
 284 // Strip NumPrefix level of directory name from PathNameStr. If the number of
 285 // directory separators is less than NumPrefix, strip all the directories and
 286 // leave base file name only.
 287 static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
 288   uint32_t Count = NumPrefix;
 289   uint32_t Pos = 0, LastPos = 0;
 290   for (const auto &CI : PathNameStr) {
 291     ++Pos;
 292     if (llvm::sys::path::is_separator(CI)) {
 293       LastPos = Pos;
 294       --Count;
 295     }
 296     if (Count == 0)
 297       break;
 298   }
 299   return PathNameStr.substr(LastPos);
 300 }
 301
 302 static StringRef getStrippedSourceFileName(const GlobalObject &GO) {
 303   StringRef FileName(GO.getParent()->getSourceFileName());
 304   uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
 305   if (StripLevel < StaticFuncStripDirNamePrefix)
 306     StripLevel = StaticFuncStripDirNamePrefix;
 307   if (StripLevel)
 308     FileName = stripDirPrefix(FileName, StripLevel);
 309   return FileName;
 310 }
 311
 312 // The PGO name has the format [<filepath>;]<mangled-name> where <filepath>; is
 313 // provided if linkage is local and is used to discriminate possibly identical
 314 // mangled names. ";" is used because it is unlikely to be found in either
 315 // <filepath> or <mangled-name>.
 316 //
 317 // Older compilers used getPGOFuncName() which has the format
 318 // [<filepath>:]<mangled-name>. This caused trouble for Objective-C functions
 319 // which commonly have :'s in their names. We still need to compute this name to
 320 // lookup functions from profiles built by older compilers.
 321 static std::string
 322 getIRPGONameForGlobalObject(const GlobalObject &GO,
 323                             GlobalValue::LinkageTypes Linkage,
 324                             StringRef FileName) {
 325   return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName);
 326 }
 327
 328 static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) {
 329   if (MD != nullptr) {
 330     StringRef S = cast<MDString>(MD->getOperand(0))->getString();
 331     return S.str();
 332   }
 333   return {};
 334 }
 335
 336 // Returns the PGO object name. This function has some special handling
 337 // when called in LTO optimization. The following only applies when calling in
 338 // LTO passes (when \c InLTO is true): LTO's internalization privatizes many
 339 // global linkage symbols. This happens after value profile annotation, but
 340 // those internal linkage functions should not have a source prefix.
 341 // Additionally, for ThinLTO mode, exported internal functions are promoted
 342 // and renamed. We need to ensure that the original internal PGO name is
 343 // used when computing the GUID that is compared against the profiled GUIDs.
 344 // To differentiate compiler generated internal symbols from original ones,
 345 // PGOFuncName meta data are created and attached to the original internal
 346 // symbols in the value profile annotation step
 347 // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta
 348 // data, its original linkage must be non-internal.
 349 static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO,
 350                                       MDNode *PGONameMetadata) {
 351   if (!InLTO) {
 352     auto FileName = getStrippedSourceFileName(GO);
 353     return getIRPGONameForGlobalObject(GO, GO.getLinkage(), FileName);
 354   }
 355
 356   // In LTO mode (when InLTO is true), first check if there is a meta data.
 357   if (auto IRPGOFuncName = lookupPGONameFromMetadata(PGONameMetadata))
 358     return *IRPGOFuncName;
 359
 360   // If there is no meta data, the function must be a global before the value
 361   // profile annotation pass. Its current linkage may be internal if it is
 362   // internalized in LTO mode.
 363   return getIRPGONameForGlobalObject(GO, GlobalValue::ExternalLinkage, "");
 364 }
 365
 366 // Returns the IRPGO function name and does special handling when called
 367 // in LTO optimization. See the comments of `getIRPGOObjectName` for details.
 368 std::string getIRPGOFuncName(const Function &F, bool InLTO) {
 369   return getIRPGOObjectName(F, InLTO, getPGOFuncNameMetadata(F));
 370 }
 371
 372 // Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
 373 // for front-end (Clang, etc) instrumentation.
 374 // The implementation is kept for profile matching from older profiles.
 375 // This is similar to `getIRPGOFuncName` except that this function calls
 376 // 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls
 377 // 'getIRPGONameForGlobalObject'. See the difference between two callees in the
 378 // comments of `getIRPGONameForGlobalObject`.
 379 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
 380   if (!InLTO) {
 381     auto FileName = getStrippedSourceFileName(F);
 382     return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
 383   }
 384
 385   // In LTO mode (when InLTO is true), first check if there is a meta data.
 386   if (auto PGOFuncName = lookupPGONameFromMetadata(getPGOFuncNameMetadata(F)))
 387     return *PGOFuncName;
 388
 389   // If there is no meta data, the function must be a global before the value
 390   // profile annotation pass. Its current linkage may be internal if it is
 391   // internalized in LTO mode.
 392   return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, "");
 393 }
 394
 395 std::string getPGOName(const GlobalVariable &V, bool InLTO) {
 396   // PGONameMetadata should be set by compiler at profile use time
 397   // and read by symtab creation to look up symbols corresponding to
 398   // a MD5 hash.
 399   return getIRPGOObjectName(V, InLTO, V.getMetadata(getPGONameMetadataName()));
 400 }
 401
 402 // See getIRPGOObjectName() for a discription of the format.
 403 std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName) {
 404   auto [FileName, MangledName] = IRPGOName.split(GlobalIdentifierDelimiter);
 405   if (MangledName.empty())
 406     return std::make_pair(StringRef(), IRPGOName);
 407   return std::make_pair(FileName, MangledName);
 408 }
 409
 410 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
 411   if (FileName.empty())
 412     return PGOFuncName;
 413   // Drop the file name including ':' or ';'. See getIRPGONameForGlobalObject as
 414   // well.
 415   if (PGOFuncName.starts_with(FileName))
 416     PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
 417   return PGOFuncName;
 418 }
 419
 420 // \p FuncName is the string used as profile lookup key for the function. A
 421 // symbol is created to hold the name. Return the legalized symbol name.
 422 std::string getPGOFuncNameVarName(StringRef FuncName,
 423                                   GlobalValue::LinkageTypes Linkage) {
 424   std::string VarName = std::string(getInstrProfNameVarPrefix());
 425   VarName += FuncName;
 426
 427   if (!GlobalValue::isLocalLinkage(Linkage))
 428     return VarName;
 429
 430   // Now fix up illegal chars in local VarName that may upset the assembler.
 431   const char InvalidChars[] = "-:;<>/\"'";
 432   size_t FoundPos = VarName.find_first_of(InvalidChars);
 433   while (FoundPos != std::string::npos) {
 434     VarName[FoundPos] = '_';
 435     FoundPos = VarName.find_first_of(InvalidChars, FoundPos + 1);
 436   }
 437   return VarName;
 438 }
 439
 440 bool isGPUProfTarget(const Module &M) {
 441   const auto &T = Triple(M.getTargetTriple());
 442   return T.isAMDGPU() || T.isNVPTX();
 443 }
 444
 445 void setPGOFuncVisibility(Module &M, GlobalVariable *FuncNameVar) {
 446   // If the target is a GPU, make the symbol protected so it can
 447   // be read from the host device
 448   if (isGPUProfTarget(M))
 449     FuncNameVar->setVisibility(GlobalValue::ProtectedVisibility);
 450   // Hide the symbol so that we correctly get a copy for each executable.
 451   else if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
 452     FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
 453 }
 454
 455 GlobalVariable *createPGOFuncNameVar(Module &M,
 456                                      GlobalValue::LinkageTypes Linkage,
 457                                      StringRef PGOFuncName) {
 458   // Ensure profiling variables on GPU are visible to be read from host
 459   if (isGPUProfTarget(M))
 460     Linkage = GlobalValue::ExternalLinkage;
 461   // We generally want to match the function's linkage, but available_externally
 462   // and extern_weak both have the wrong semantics, and anything that doesn't
 463   // need to link across compilation units doesn't need to be visible at all.
 464   else if (Linkage == GlobalValue::ExternalWeakLinkage)
 465     Linkage = GlobalValue::LinkOnceAnyLinkage;
 466   else if (Linkage == GlobalValue::AvailableExternallyLinkage)
 467     Linkage = GlobalValue::LinkOnceODRLinkage;
 468   else if (Linkage == GlobalValue::InternalLinkage ||
 469            Linkage == GlobalValue::ExternalLinkage)
 470     Linkage = GlobalValue::PrivateLinkage;
 471
 472   auto *Value =
 473       ConstantDataArray::getString(M.getContext(), PGOFuncName, false);
 474   auto *FuncNameVar =
 475       new GlobalVariable(M, Value->getType(), true, Linkage, Value,
 476                          getPGOFuncNameVarName(PGOFuncName, Linkage));
 477
 478   setPGOFuncVisibility(M, FuncNameVar);
 479   return FuncNameVar;
 480 }
 481
 482 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) {
 483   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName);
 484 }
 485
 486 Error InstrProfSymtab::create(Module &M, bool InLTO, bool AddCanonical) {
 487   for (Function &F : M) {
 488     // Function may not have a name: like using asm("") to overwrite the name.
 489     // Ignore in this case.
 490     if (!F.hasName())
 491       continue;
 492     if (Error E = addFuncWithName(F, getIRPGOFuncName(F, InLTO), AddCanonical))
 493       return E;
 494     // Also use getPGOFuncName() so that we can find records from older profiles
 495     if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO), AddCanonical))
 496       return E;
 497   }
 498
 499   SmallVector<MDNode *, 2> Types;
 500   for (GlobalVariable &G : M.globals()) {
 501     if (!G.hasName() || !G.hasMetadata(LLVMContext::MD_type))
 502       continue;
 503     if (Error E = addVTableWithName(G, getPGOName(G, InLTO)))
 504       return E;
 505   }
 506
 507   Sorted = false;
 508   finalizeSymtab();
 509   return Error::success();
 510 }
 511
 512 Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable,
 513                                          StringRef VTablePGOName) {
 514   auto NameToGUIDMap = [&](StringRef Name) -> Error {
 515     if (Error E = addSymbolName(Name))
 516       return E;
 517
 518     bool Inserted = true;
 519     std::tie(std::ignore, Inserted) =
 520         MD5VTableMap.try_emplace(GlobalValue::getGUID(Name), &VTable);
 521     if (!Inserted)
 522       LLVM_DEBUG(dbgs() << "GUID conflict within one module");
 523     return Error::success();
 524   };
 525   if (Error E = NameToGUIDMap(VTablePGOName))
 526     return E;
 527
 528   StringRef CanonicalName = getCanonicalName(VTablePGOName);
 529   if (CanonicalName != VTablePGOName)
 530     return NameToGUIDMap(CanonicalName);
 531
 532   return Error::success();
 533 }
 534
 535 /// \c NameStrings is a string composed of one of more possibly encoded
 536 /// sub-strings. The substrings are separated by 0 or more zero bytes. This
 537 /// method decodes the string and calls `NameCallback` for each substring.
 538 static Error
 539 readAndDecodeStrings(StringRef NameStrings,
 540                      std::function<Error(StringRef)> NameCallback) {
 541   const uint8_t *P = NameStrings.bytes_begin();
 542   const uint8_t *EndP = NameStrings.bytes_end();
 543   while (P < EndP) {
 544     uint32_t N;
 545     uint64_t UncompressedSize = decodeULEB128(P, &N);
 546     P += N;
 547     uint64_t CompressedSize = decodeULEB128(P, &N);
 548     P += N;
 549     const bool IsCompressed = (CompressedSize != 0);
 550     SmallVector<uint8_t, 128> UncompressedNameStrings;
 551     StringRef NameStrings;
 552     if (IsCompressed) {
 553       if (!llvm::compression::zlib::isAvailable())
 554         return make_error<InstrProfError>(instrprof_error::zlib_unavailable);
 555
 556       if (Error E = compression::zlib::decompress(ArrayRef(P, CompressedSize),
 557                                                   UncompressedNameStrings,
 558                                                   UncompressedSize)) {
 559         consumeError(std::move(E));
 560         return make_error<InstrProfError>(instrprof_error::uncompress_failed);
 561       }
 562       P += CompressedSize;
 563       NameStrings = toStringRef(UncompressedNameStrings);
 564     } else {
 565       NameStrings =
 566           StringRef(reinterpret_cast<const char *>(P), UncompressedSize);
 567       P += UncompressedSize;
 568     }
 569     // Now parse the name strings.
 570     SmallVector<StringRef, 0> Names;
 571     NameStrings.split(Names, getInstrProfNameSeparator());
 572     for (StringRef &Name : Names)
 573       if (Error E = NameCallback(Name))
 574         return E;
 575
 576     while (P < EndP && *P == 0)
 577       P++;
 578   }
 579   return Error::success();
 580 }
 581
 582 Error InstrProfSymtab::create(StringRef NameStrings) {
 583   return readAndDecodeStrings(
 584       NameStrings,
 585       std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
 586 }
 587
 588 Error InstrProfSymtab::create(StringRef FuncNameStrings,
 589                               StringRef VTableNameStrings) {
 590   if (Error E = readAndDecodeStrings(FuncNameStrings,
 591                                      std::bind(&InstrProfSymtab::addFuncName,
 592                                                this, std::placeholders::_1)))
 593     return E;
 594
 595   return readAndDecodeStrings(
 596       VTableNameStrings,
 597       std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
 598 }
 599
 600 Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
 601     StringRef CompressedVTableStrings) {
 602   return readAndDecodeStrings(
 603       CompressedVTableStrings,
 604       std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
 605 }
 606
 607 StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
 608   // In ThinLTO, local function may have been promoted to global and have
 609   // suffix ".llvm." added to the function name. We need to add the
 610   // stripped function name to the symbol table so that we can find a match
 611   // from profile.
 612   //
 613   // ".__uniq." suffix is used to differentiate internal linkage functions in
 614   // different modules and should be kept. This is the only suffix with the
 615   // pattern ".xxx" which is kept before matching, other suffixes similar as
 616   // ".llvm." will be stripped.
 617   const std::string UniqSuffix = ".__uniq.";
 618   size_t Pos = PGOName.find(UniqSuffix);
 619   if (Pos != StringRef::npos)
 620     Pos += UniqSuffix.length();
 621   else
 622     Pos = 0;
 623
 624   // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from
 625   // the beginning.
 626   Pos = PGOName.find('.', Pos);
 627   if (Pos != StringRef::npos && Pos != 0)
 628     return PGOName.substr(0, Pos);
 629
 630   return PGOName;
 631 }
 632
 633 Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName,
 634                                        bool AddCanonical) {
 635   auto NameToGUIDMap = [&](StringRef Name) -> Error {
 636     if (Error E = addFuncName(Name))
 637       return E;
 638     MD5FuncMap.emplace_back(Function::getGUID(Name), &F);
 639     return Error::success();
 640   };
 641   if (Error E = NameToGUIDMap(PGOFuncName))
 642     return E;
 643
 644   if (!AddCanonical)
 645     return Error::success();
 646
 647   StringRef CanonicalFuncName = getCanonicalName(PGOFuncName);
 648   if (CanonicalFuncName != PGOFuncName)
 649     return NameToGUIDMap(CanonicalFuncName);
 650
 651   return Error::success();
 652 }
 653
 654 uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
 655   // Given a runtime address, look up the hash value in the interval map, and
 656   // fallback to value 0 if a hash value is not found.
 657   return VTableAddrMap.lookup(Address, 0);
 658 }
 659
 660 uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
 661   finalizeSymtab();
 662   auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
 663     return A.first < Address;
 664   });
 665   // Raw function pointer collected by value profiler may be from
 666   // external functions that are not instrumented. They won't have
 667   // mapping data to be used by the deserializer. Force the value to
 668   // be 0 in this case.
 669   if (It != AddrToMD5Map.end() && It->first == Address)
 670     return (uint64_t)It->second;
 671   return 0;
 672 }
 673
 674 void InstrProfSymtab::dumpNames(raw_ostream &OS) const {
 675   SmallVector<StringRef, 0> Sorted(NameTab.keys());
 676   llvm::sort(Sorted);
 677   for (StringRef S : Sorted)
 678     OS << S << '\n';
 679 }
 680
 681 Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
 682                                      bool DoCompression, std::string &Result) {
 683   assert(!NameStrs.empty() && "No name data to emit");
 684
 685   uint8_t Header[20], *P = Header;
 686   std::string UncompressedNameStrings =
 687       join(NameStrs.begin(), NameStrs.end(), getInstrProfNameSeparator());
 688
 689   assert(StringRef(UncompressedNameStrings)
 690                  .count(getInstrProfNameSeparator()) == (NameStrs.size() - 1) &&
 691          "PGO name is invalid (contains separator token)");
 692
 693   unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);
 694   P += EncLen;
 695
 696   auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {
 697     EncLen = encodeULEB128(CompressedLen, P);
 698     P += EncLen;
 699     char *HeaderStr = reinterpret_cast<char *>(&Header[0]);
 700     unsigned HeaderLen = P - &Header[0];
 701     Result.append(HeaderStr, HeaderLen);
 702     Result += InputStr;
 703     return Error::success();
 704   };
 705
 706   if (!DoCompression) {
 707     return WriteStringToResult(0, UncompressedNameStrings);
 708   }
 709
 710   SmallVector<uint8_t, 128> CompressedNameStrings;
 711   compression::zlib::compress(arrayRefFromStringRef(UncompressedNameStrings),
 712                               CompressedNameStrings,
 713                               compression::zlib::BestSizeCompression);
 714
 715   return WriteStringToResult(CompressedNameStrings.size(),
 716                              toStringRef(CompressedNameStrings));
 717 }
 718
 719 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) {
 720   auto *Arr = cast<ConstantDataArray>(NameVar->getInitializer());
 721   StringRef NameStr =
 722       Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
 723   return NameStr;
 724 }
 725
 726 Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
 727                                 std::string &Result, bool DoCompression) {
 728   std::vector<std::string> NameStrs;
 729   for (auto *NameVar : NameVars) {
 730     NameStrs.push_back(std::string(getPGOFuncNameVarInitializer(NameVar)));
 731   }
 732   return collectGlobalObjectNameStrings(
 733       NameStrs, compression::zlib::isAvailable() && DoCompression, Result);
 734 }
 735
 736 Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
 737                            std::string &Result, bool DoCompression) {
 738   std::vector<std::string> VTableNameStrs;
 739   for (auto *VTable : VTables)
 740     VTableNameStrs.push_back(getPGOName(*VTable));
 741   return collectGlobalObjectNameStrings(
 742       VTableNameStrs, compression::zlib::isAvailable() && DoCompression,
 743       Result);
 744 }
 745
 746 void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
 747   uint64_t FuncSum = 0;
 748   Sum.NumEntries += Counts.size();
 749   for (uint64_t Count : Counts)
 750     FuncSum += Count;
 751   Sum.CountSum += FuncSum;
 752
 753   for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {
 754     uint64_t KindSum = 0;
 755     uint32_t NumValueSites = getNumValueSites(VK);
 756     for (size_t I = 0; I < NumValueSites; ++I) {
 757       for (const auto &V : getValueArrayForSite(VK, I))
 758         KindSum += V.Count;
 759     }
 760     Sum.ValueCounts[VK] += KindSum;
 761   }
 762 }
 763
 764 void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input,
 765                                        uint32_t ValueKind,
 766                                        OverlapStats &Overlap,
 767                                        OverlapStats &FuncLevelOverlap) {
 768   this->sortByTargetValues();
 769   Input.sortByTargetValues();
 770   double Score = 0.0f, FuncLevelScore = 0.0f;
 771   auto I = ValueData.begin();
 772   auto IE = ValueData.end();
 773   auto J = Input.ValueData.begin();
 774   auto JE = Input.ValueData.end();
 775   while (I != IE && J != JE) {
 776     if (I->Value == J->Value) {
 777       Score += OverlapStats::score(I->Count, J->Count,
 778                                    Overlap.Base.ValueCounts[ValueKind],
 779                                    Overlap.Test.ValueCounts[ValueKind]);
 780       FuncLevelScore += OverlapStats::score(
 781           I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],
 782           FuncLevelOverlap.Test.ValueCounts[ValueKind]);
 783       ++I;
 784     } else if (I->Value < J->Value) {
 785       ++I;
 786       continue;
 787     }
 788     ++J;
 789   }
 790   Overlap.Overlap.ValueCounts[ValueKind] += Score;
 791   FuncLevelOverlap.Overlap.ValueCounts[ValueKind] += FuncLevelScore;
 792 }
 793
 794 // Return false on mismatch.
 795 void InstrProfRecord::overlapValueProfData(uint32_t ValueKind,
 796                                            InstrProfRecord &Other,
 797                                            OverlapStats &Overlap,
 798                                            OverlapStats &FuncLevelOverlap) {
 799   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
 800   assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));
 801   if (!ThisNumValueSites)
 802     return;
 803
 804   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
 805       getOrCreateValueSitesForKind(ValueKind);
 806   MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
 807       Other.getValueSitesForKind(ValueKind);
 808   for (uint32_t I = 0; I < ThisNumValueSites; I++)
 809     ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,
 810                                FuncLevelOverlap);
 811 }
 812
 813 void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
 814                               OverlapStats &FuncLevelOverlap,
 815                               uint64_t ValueCutoff) {
 816   // FuncLevel CountSum for other should already computed and nonzero.
 817   assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
 818   accumulateCounts(FuncLevelOverlap.Base);
 819   bool Mismatch = (Counts.size() != Other.Counts.size());
 820
 821   // Check if the value profiles mismatch.
 822   if (!Mismatch) {
 823     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
 824       uint32_t ThisNumValueSites = getNumValueSites(Kind);
 825       uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);
 826       if (ThisNumValueSites != OtherNumValueSites) {
 827         Mismatch = true;
 828         break;
 829       }
 830     }
 831   }
 832   if (Mismatch) {
 833     Overlap.addOneMismatch(FuncLevelOverlap.Test);
 834     return;
 835   }
 836
 837   // Compute overlap for value counts.
 838   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
 839     overlapValueProfData(Kind, Other, Overlap, FuncLevelOverlap);
 840
 841   double Score = 0.0;
 842   uint64_t MaxCount = 0;
 843   // Compute overlap for edge counts.
 844   for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
 845     Score += OverlapStats::score(Counts[I], Other.Counts[I],
 846                                  Overlap.Base.CountSum, Overlap.Test.CountSum);
 847     MaxCount = std::max(Other.Counts[I], MaxCount);
 848   }
 849   Overlap.Overlap.CountSum += Score;
 850   Overlap.Overlap.NumEntries += 1;
 851
 852   if (MaxCount >= ValueCutoff) {
 853     double FuncScore = 0.0;
 854     for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)
 855       FuncScore += OverlapStats::score(Counts[I], Other.Counts[I],
 856                                        FuncLevelOverlap.Base.CountSum,
 857                                        FuncLevelOverlap.Test.CountSum);
 858     FuncLevelOverlap.Overlap.CountSum = FuncScore;
 859     FuncLevelOverlap.Overlap.NumEntries = Other.Counts.size();
 860     FuncLevelOverlap.Valid = true;
 861   }
 862 }
 863
 864 void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input,
 865                                      uint64_t Weight,
 866                                      function_ref<void(instrprof_error)> Warn) {
 867   this->sortByTargetValues();
 868   Input.sortByTargetValues();
 869   auto I = ValueData.begin();
 870   auto IE = ValueData.end();
 871   std::vector<InstrProfValueData> Merged;
 872   Merged.reserve(std::max(ValueData.size(), Input.ValueData.size()));
 873   for (const InstrProfValueData &J : Input.ValueData) {
 874     while (I != IE && I->Value < J.Value) {
 875       Merged.push_back(*I);
 876       ++I;
 877     }
 878     if (I != IE && I->Value == J.Value) {
 879       bool Overflowed;
 880       I->Count = SaturatingMultiplyAdd(J.Count, Weight, I->Count, &Overflowed);
 881       if (Overflowed)
 882         Warn(instrprof_error::counter_overflow);
 883       Merged.push_back(*I);
 884       ++I;
 885       continue;
 886     }
 887     Merged.push_back(J);
 888   }
 889   Merged.insert(Merged.end(), I, IE);
 890   ValueData = std::move(Merged);
 891 }
 892
 893 void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D,
 894                                      function_ref<void(instrprof_error)> Warn) {
 895   for (InstrProfValueData &I : ValueData) {
 896     bool Overflowed;
 897     I.Count = SaturatingMultiply(I.Count, N, &Overflowed) / D;
 898     if (Overflowed)
 899       Warn(instrprof_error::counter_overflow);
 900   }
 901 }
 902
 903 // Merge Value Profile data from Src record to this record for ValueKind.
 904 // Scale merged value counts by \p Weight.
 905 void InstrProfRecord::mergeValueProfData(
 906     uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight,
 907     function_ref<void(instrprof_error)> Warn) {
 908   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
 909   uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
 910   if (ThisNumValueSites != OtherNumValueSites) {
 911     Warn(instrprof_error::value_site_count_mismatch);
 912     return;
 913   }
 914   if (!ThisNumValueSites)
 915     return;
 916   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
 917       getOrCreateValueSitesForKind(ValueKind);
 918   MutableArrayRef<InstrProfValueSiteRecord> OtherSiteRecords =
 919       Src.getValueSitesForKind(ValueKind);
 920   for (uint32_t I = 0; I < ThisNumValueSites; I++)
 921     ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);
 922 }
 923
 924 void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight,
 925                             function_ref<void(instrprof_error)> Warn) {
 926   // If the number of counters doesn't match we either have bad data
 927   // or a hash collision.
 928   if (Counts.size() != Other.Counts.size()) {
 929     Warn(instrprof_error::count_mismatch);
 930     return;
 931   }
 932
 933   // Special handling of the first count as the PseudoCount.
 934   CountPseudoKind OtherKind = Other.getCountPseudoKind();
 935   CountPseudoKind ThisKind = getCountPseudoKind();
 936   if (OtherKind != NotPseudo || ThisKind != NotPseudo) {
 937     // We don't allow the merge of a profile with pseudo counts and
 938     // a normal profile (i.e. without pesudo counts).
 939     // Profile supplimenation should be done after the profile merge.
 940     if (OtherKind == NotPseudo || ThisKind == NotPseudo) {
 941       Warn(instrprof_error::count_mismatch);
 942       return;
 943     }
 944     if (OtherKind == PseudoHot || ThisKind == PseudoHot)
 945       setPseudoCount(PseudoHot);
 946     else
 947       setPseudoCount(PseudoWarm);
 948     return;
 949   }
 950
 951   for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
 952     bool Overflowed;
 953     uint64_t Value =
 954         SaturatingMultiplyAdd(Other.Counts[I], Weight, Counts[I], &Overflowed);
 955     if (Value > getInstrMaxCountValue()) {
 956       Value = getInstrMaxCountValue();
 957       Overflowed = true;
 958     }
 959     Counts[I] = Value;
 960     if (Overflowed)
 961       Warn(instrprof_error::counter_overflow);
 962   }
 963
 964   // If the number of bitmap bytes doesn't match we either have bad data
 965   // or a hash collision.
 966   if (BitmapBytes.size() != Other.BitmapBytes.size()) {
 967     Warn(instrprof_error::bitmap_mismatch);
 968     return;
 969   }
 970
 971   // Bitmap bytes are merged by simply ORing them together.
 972   for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) {
 973     BitmapBytes[I] = Other.BitmapBytes[I] | BitmapBytes[I];
 974   }
 975
 976   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
 977     mergeValueProfData(Kind, Other, Weight, Warn);
 978 }
 979
 980 void InstrProfRecord::scaleValueProfData(
 981     uint32_t ValueKind, uint64_t N, uint64_t D,
 982     function_ref<void(instrprof_error)> Warn) {
 983   for (auto &R : getValueSitesForKind(ValueKind))
 984     R.scale(N, D, Warn);
 985 }
 986
 987 void InstrProfRecord::scale(uint64_t N, uint64_t D,
 988                             function_ref<void(instrprof_error)> Warn) {
 989   assert(D != 0 && "D cannot be 0");
 990   for (auto &Count : this->Counts) {
 991     bool Overflowed;
 992     Count = SaturatingMultiply(Count, N, &Overflowed) / D;
 993     if (Count > getInstrMaxCountValue()) {
 994       Count = getInstrMaxCountValue();
 995       Overflowed = true;
 996     }
 997     if (Overflowed)
 998       Warn(instrprof_error::counter_overflow);
 999   }
1000   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1001     scaleValueProfData(Kind, N, D, Warn);
1002 }
1003
1004 // Map indirect call target name hash to name string.
1005 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
1006                                      InstrProfSymtab *SymTab) {
1007   if (!SymTab)
1008     return Value;
1009
1010   if (ValueKind == IPVK_IndirectCallTarget)
1011     return SymTab->getFunctionHashFromAddress(Value);
1012
1013   if (ValueKind == IPVK_VTableTarget)
1014     return SymTab->getVTableHashFromAddress(Value);
1015
1016   return Value;
1017 }
1018
1019 void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
1020                                    ArrayRef<InstrProfValueData> VData,
1021                                    InstrProfSymtab *ValueMap) {
1022   // Remap values.
1023   std::vector<InstrProfValueData> RemappedVD;
1024   RemappedVD.reserve(VData.size());
1025   for (const auto &V : VData) {
1026     uint64_t NewValue = remapValue(V.Value, ValueKind, ValueMap);
1027     RemappedVD.push_back({NewValue, V.Count});
1028   }
1029
1030   std::vector<InstrProfValueSiteRecord> &ValueSites =
1031       getOrCreateValueSitesForKind(ValueKind);
1032   assert(ValueSites.size() == Site);
1033
1034   // Add a new value site with remapped value profiling data.
1035   ValueSites.emplace_back(std::move(RemappedVD));
1036 }
1037
1038 void TemporalProfTraceTy::createBPFunctionNodes(
1039     ArrayRef<TemporalProfTraceTy> Traces, std::vector<BPFunctionNode> &Nodes,
1040     bool RemoveOutlierUNs) {
1041   using IDT = BPFunctionNode::IDT;
1042   using UtilityNodeT = BPFunctionNode::UtilityNodeT;
1043   UtilityNodeT MaxUN = 0;
1044   DenseMap<IDT, size_t> IdToFirstTimestamp;
1045   DenseMap<IDT, UtilityNodeT> IdToFirstUN;
1046   DenseMap<IDT, SmallVector<UtilityNodeT>> IdToUNs;
1047   // TODO: We need to use the Trace.Weight field to give more weight to more
1048   // important utilities
1049   for (auto &Trace : Traces) {
1050     size_t CutoffTimestamp = 1;
1051     for (size_t Timestamp = 0; Timestamp < Trace.FunctionNameRefs.size();
1052          Timestamp++) {
1053       IDT Id = Trace.FunctionNameRefs[Timestamp];
1054       auto [It, WasInserted] = IdToFirstTimestamp.try_emplace(Id, Timestamp);
1055       if (!WasInserted)
1056         It->getSecond() = std::min<size_t>(It->getSecond(), Timestamp);
1057       if (Timestamp >= CutoffTimestamp) {
1058         ++MaxUN;
1059         CutoffTimestamp = 2 * Timestamp;
1060       }
1061       IdToFirstUN.try_emplace(Id, MaxUN);
1062     }
1063     for (auto &[Id, FirstUN] : IdToFirstUN)
1064       for (auto UN = FirstUN; UN <= MaxUN; ++UN)
1065         IdToUNs[Id].push_back(UN);
1066     ++MaxUN;
1067     IdToFirstUN.clear();
1068   }
1069
1070   if (RemoveOutlierUNs) {
1071     DenseMap<UtilityNodeT, unsigned> UNFrequency;
1072     for (auto &[Id, UNs] : IdToUNs)
1073       for (auto &UN : UNs)
1074         ++UNFrequency[UN];
1075     // Filter out utility nodes that are too infrequent or too prevalent to make
1076     // BalancedPartitioning more effective.
1077     for (auto &[Id, UNs] : IdToUNs)
1078       llvm::erase_if(UNs, [&](auto &UN) {
1079         return UNFrequency[UN] <= 1 || 2 * UNFrequency[UN] > IdToUNs.size();
1080       });
1081   }
1082
1083   for (auto &[Id, UNs] : IdToUNs)
1084     Nodes.emplace_back(Id, UNs);
1085
1086   // Since BalancedPartitioning is sensitive to the initial order, we explicitly
1087   // order nodes by their earliest timestamp.
1088   llvm::sort(Nodes, [&](auto &L, auto &R) {
1089     return std::make_pair(IdToFirstTimestamp[L.Id], L.Id) <
1090            std::make_pair(IdToFirstTimestamp[R.Id], R.Id);
1091   });
1092 }
1093
1094 #define INSTR_PROF_COMMON_API_IMPL
1095 #include "llvm/ProfileData/InstrProfData.inc"
1096
1097 /*!
1098  * ValueProfRecordClosure Interface implementation for  InstrProfRecord
1099  *  class. These C wrappers are used as adaptors so that C++ code can be
1100  *  invoked as callbacks.
1101  */
1102 uint32_t getNumValueKindsInstrProf(const void *Record) {
1103   return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
1104 }
1105
1106 uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
1107   return reinterpret_cast<const InstrProfRecord *>(Record)
1108       ->getNumValueSites(VKind);
1109 }
1110
1111 uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
1112   return reinterpret_cast<const InstrProfRecord *>(Record)
1113       ->getNumValueData(VKind);
1114 }
1115
1116 uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
1117                                          uint32_t S) {
1118   const auto *IPR = reinterpret_cast<const InstrProfRecord *>(R);
1119   return IPR->getValueArrayForSite(VK, S).size();
1120 }
1121
1122 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
1123                               uint32_t K, uint32_t S) {
1124   const auto *IPR = reinterpret_cast<const InstrProfRecord *>(R);
1125   llvm::copy(IPR->getValueArrayForSite(K, S), Dst);
1126 }
1127
1128 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
1129   ValueProfData *VD =
1130       (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
1131   memset(VD, 0, TotalSizeInBytes);
1132   return VD;
1133 }
1134
1135 static ValueProfRecordClosure InstrProfRecordClosure = {
1136     nullptr,
1137     getNumValueKindsInstrProf,
1138     getNumValueSitesInstrProf,
1139     getNumValueDataInstrProf,
1140     getNumValueDataForSiteInstrProf,
1141     nullptr,
1142     getValueForSiteInstrProf,
1143     allocValueProfDataInstrProf};
1144
1145 // Wrapper implementation using the closure mechanism.
1146 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
1147   auto Closure = InstrProfRecordClosure;
1148   Closure.Record = &Record;
1149   return getValueProfDataSize(&Closure);
1150 }
1151
1152 // Wrapper implementation using the closure mechanism.
1153 std::unique_ptr<ValueProfData>
1154 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
1155   InstrProfRecordClosure.Record = &Record;
1156
1157   std::unique_ptr<ValueProfData> VPD(
1158       serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
1159   return VPD;
1160 }
1161
1162 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
1163                                     InstrProfSymtab *SymTab) {
1164   Record.reserveSites(Kind, NumValueSites);
1165
1166   InstrProfValueData *ValueData = getValueProfRecordValueData(this);
1167   for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
1168     uint8_t ValueDataCount = this->SiteCountArray[VSite];
1169     ArrayRef<InstrProfValueData> VDs(ValueData, ValueDataCount);
1170     Record.addValueData(Kind, VSite, VDs, SymTab);
1171     ValueData += ValueDataCount;
1172   }
1173 }
1174
1175 // For writing/serializing,  Old is the host endianness, and  New is
1176 // byte order intended on disk. For Reading/deserialization, Old
1177 // is the on-disk source endianness, and New is the host endianness.
1178 void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) {
1179   using namespace support;
1180
1181   if (Old == New)
1182     return;
1183
1184   if (llvm::endianness::native != Old) {
1185     sys::swapByteOrder<uint32_t>(NumValueSites);
1186     sys::swapByteOrder<uint32_t>(Kind);
1187   }
1188   uint32_t ND = getValueProfRecordNumValueData(this);
1189   InstrProfValueData *VD = getValueProfRecordValueData(this);
1190
1191   // No need to swap byte array: SiteCountArrray.
1192   for (uint32_t I = 0; I < ND; I++) {
1193     sys::swapByteOrder<uint64_t>(VD[I].Value);
1194     sys::swapByteOrder<uint64_t>(VD[I].Count);
1195   }
1196   if (llvm::endianness::native == Old) {
1197     sys::swapByteOrder<uint32_t>(NumValueSites);
1198     sys::swapByteOrder<uint32_t>(Kind);
1199   }
1200 }
1201
1202 void ValueProfData::deserializeTo(InstrProfRecord &Record,
1203                                   InstrProfSymtab *SymTab) {
1204   if (NumValueKinds == 0)
1205     return;
1206
1207   ValueProfRecord *VR = getFirstValueProfRecord(this);
1208   for (uint32_t K = 0; K < NumValueKinds; K++) {
1209     VR->deserializeTo(Record, SymTab);
1210     VR = getValueProfRecordNext(VR);
1211   }
1212 }
1213
1214 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
1215   return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
1216                                             ValueProfData());
1217 }
1218
1219 Error ValueProfData::checkIntegrity() {
1220   if (NumValueKinds > IPVK_Last + 1)
1221     return make_error<InstrProfError>(
1222         instrprof_error::malformed, "number of value profile kinds is invalid");
1223   // Total size needs to be multiple of quadword size.
1224   if (TotalSize % sizeof(uint64_t))
1225     return make_error<InstrProfError>(
1226         instrprof_error::malformed, "total size is not multiples of quardword");
1227
1228   ValueProfRecord *VR = getFirstValueProfRecord(this);
1229   for (uint32_t K = 0; K < this->NumValueKinds; K++) {
1230     if (VR->Kind > IPVK_Last)
1231       return make_error<InstrProfError>(instrprof_error::malformed,
1232                                         "value kind is invalid");
1233     VR = getValueProfRecordNext(VR);
1234     if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
1235       return make_error<InstrProfError>(
1236           instrprof_error::malformed,
1237           "value profile address is greater than total size");
1238   }
1239   return Error::success();
1240 }
1241
1242 Expected<std::unique_ptr<ValueProfData>>
1243 ValueProfData::getValueProfData(const unsigned char *D,
1244                                 const unsigned char *const BufferEnd,
1245                                 llvm::endianness Endianness) {
1246   using namespace support;
1247
1248   if (D + sizeof(ValueProfData) > BufferEnd)
1249     return make_error<InstrProfError>(instrprof_error::truncated);
1250
1251   const unsigned char *Header = D;
1252   uint32_t TotalSize = endian::readNext<uint32_t>(Header, Endianness);
1253
1254   if (D + TotalSize > BufferEnd)
1255     return make_error<InstrProfError>(instrprof_error::too_large);
1256
1257   std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
1258   memcpy(VPD.get(), D, TotalSize);
1259   // Byte swap.
1260   VPD->swapBytesToHost(Endianness);
1261
1262   Error E = VPD->checkIntegrity();
1263   if (E)
1264     return std::move(E);
1265
1266   return std::move(VPD);
1267 }
1268
1269 void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {
1270   using namespace support;
1271
1272   if (Endianness == llvm::endianness::native)
1273     return;
1274
1275   sys::swapByteOrder<uint32_t>(TotalSize);
1276   sys::swapByteOrder<uint32_t>(NumValueKinds);
1277
1278   ValueProfRecord *VR = getFirstValueProfRecord(this);
1279   for (uint32_t K = 0; K < NumValueKinds; K++) {
1280     VR->swapBytes(Endianness, llvm::endianness::native);
1281     VR = getValueProfRecordNext(VR);
1282   }
1283 }
1284
1285 void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {
1286   using namespace support;
1287
1288   if (Endianness == llvm::endianness::native)
1289     return;
1290
1291   ValueProfRecord *VR = getFirstValueProfRecord(this);
1292   for (uint32_t K = 0; K < NumValueKinds; K++) {
1293     ValueProfRecord *NVR = getValueProfRecordNext(VR);
1294     VR->swapBytes(llvm::endianness::native, Endianness);
1295     VR = NVR;
1296   }
1297   sys::swapByteOrder<uint32_t>(TotalSize);
1298   sys::swapByteOrder<uint32_t>(NumValueKinds);
1299 }
1300
1301 void annotateValueSite(Module &M, Instruction &Inst,
1302                        const InstrProfRecord &InstrProfR,
1303                        InstrProfValueKind ValueKind, uint32_t SiteIdx,
1304                        uint32_t MaxMDCount) {
1305   auto VDs = InstrProfR.getValueArrayForSite(ValueKind, SiteIdx);
1306   if (VDs.empty())
1307     return;
1308   uint64_t Sum = 0;
1309   for (const InstrProfValueData &V : VDs)
1310     Sum = SaturatingAdd(Sum, V.Count);
1311   annotateValueSite(M, Inst, VDs, Sum, ValueKind, MaxMDCount);
1312 }
1313
1314 void annotateValueSite(Module &M, Instruction &Inst,
1315                        ArrayRef<InstrProfValueData> VDs,
1316                        uint64_t Sum, InstrProfValueKind ValueKind,
1317                        uint32_t MaxMDCount) {
1318   if (VDs.empty())
1319     return;
1320   LLVMContext &Ctx = M.getContext();
1321   MDBuilder MDHelper(Ctx);
1322   SmallVector<Metadata *, 3> Vals;
1323   // Tag
1324   Vals.push_back(MDHelper.createString("VP"));
1325   // Value Kind
1326   Vals.push_back(MDHelper.createConstant(
1327       ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
1328   // Total Count
1329   Vals.push_back(
1330       MDHelper.createConstant(ConstantInt::get(Type::getInt64Ty(Ctx), Sum)));
1331
1332   // Value Profile Data
1333   uint32_t MDCount = MaxMDCount;
1334   for (const auto &VD : VDs) {
1335     Vals.push_back(MDHelper.createConstant(
1336         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Value)));
1337     Vals.push_back(MDHelper.createConstant(
1338         ConstantInt::get(Type::getInt64Ty(Ctx), VD.Count)));
1339     if (--MDCount == 0)
1340       break;
1341   }
1342   Inst.setMetadata(LLVMContext::MD_prof, MDNode::get(Ctx, Vals));
1343 }
1344
1345 MDNode *mayHaveValueProfileOfKind(const Instruction &Inst,
1346                                   InstrProfValueKind ValueKind) {
1347   MDNode *MD = Inst.getMetadata(LLVMContext::MD_prof);
1348   if (!MD)
1349     return nullptr;
1350
1351   if (MD->getNumOperands() < 5)
1352     return nullptr;
1353
1354   MDString *Tag = cast<MDString>(MD->getOperand(0));
1355   if (!Tag || Tag->getString() != "VP")
1356     return nullptr;
1357
1358   // Now check kind:
1359   ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
1360   if (!KindInt)
1361     return nullptr;
1362   if (KindInt->getZExtValue() != ValueKind)
1363     return nullptr;
1364
1365   return MD;
1366 }
1367
1368 SmallVector<InstrProfValueData, 4>
1369 getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
1370                          uint32_t MaxNumValueData, uint64_t &TotalC,
1371                          bool GetNoICPValue) {
1372   // Four inline elements seem to work well in practice.  With MaxNumValueData,
1373   // this array won't grow very big anyway.
1374   SmallVector<InstrProfValueData, 4> ValueData;
1375   MDNode *MD = mayHaveValueProfileOfKind(Inst, ValueKind);
1376   if (!MD)
1377     return ValueData;
1378   const unsigned NOps = MD->getNumOperands();
1379   // Get total count
1380   ConstantInt *TotalCInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
1381   if (!TotalCInt)
1382     return ValueData;
1383   TotalC = TotalCInt->getZExtValue();
1384
1385   ValueData.reserve((NOps - 3) / 2);
1386   for (unsigned I = 3; I < NOps; I += 2) {
1387     if (ValueData.size() >= MaxNumValueData)
1388       break;
1389     ConstantInt *Value = mdconst::dyn_extract<ConstantInt>(MD->getOperand(I));
1390     ConstantInt *Count =
1391         mdconst::dyn_extract<ConstantInt>(MD->getOperand(I + 1));
1392     if (!Value || !Count) {
1393       ValueData.clear();
1394       return ValueData;
1395     }
1396     uint64_t CntValue = Count->getZExtValue();
1397     if (!GetNoICPValue && (CntValue == NOMORE_ICP_MAGICNUM))
1398       continue;
1399     InstrProfValueData V;
1400     V.Value = Value->getZExtValue();
1401     V.Count = CntValue;
1402     ValueData.push_back(V);
1403   }
1404   return ValueData;
1405 }
1406
1407 MDNode *getPGOFuncNameMetadata(const Function &F) {
1408   return F.getMetadata(getPGOFuncNameMetadataName());
1409 }
1410
1411 static void createPGONameMetadata(GlobalObject &GO, StringRef MetadataName,
1412                                   StringRef PGOName) {
1413   // Only for internal linkage functions or global variables. The name is not
1414   // the same as PGO name for these global objects.
1415   if (GO.getName() == PGOName)
1416     return;
1417
1418   // Don't create duplicated metadata.
1419   if (GO.getMetadata(MetadataName))
1420     return;
1421
1422   LLVMContext &C = GO.getContext();
1423   MDNode *N = MDNode::get(C, MDString::get(C, PGOName));
1424   GO.setMetadata(MetadataName, N);
1425 }
1426
1427 void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) {
1428   return createPGONameMetadata(F, getPGOFuncNameMetadataName(), PGOFuncName);
1429 }
1430
1431 void createPGONameMetadata(GlobalObject &GO, StringRef PGOName) {
1432   return createPGONameMetadata(GO, getPGONameMetadataName(), PGOName);
1433 }
1434
1435 bool needsComdatForCounter(const GlobalObject &GO, const Module &M) {
1436   if (GO.hasComdat())
1437     return true;
1438
1439   if (!Triple(M.getTargetTriple()).supportsCOMDAT())
1440     return false;
1441
1442   // See createPGOFuncNameVar for more details. To avoid link errors, profile
1443   // counters for function with available_externally linkage needs to be changed
1444   // to linkonce linkage. On ELF based systems, this leads to weak symbols to be
1445   // created. Without using comdat, duplicate entries won't be removed by the
1446   // linker leading to increased data segement size and raw profile size. Even
1447   // worse, since the referenced counter from profile per-function data object
1448   // will be resolved to the common strong definition, the profile counts for
1449   // available_externally functions will end up being duplicated in raw profile
1450   // data. This can result in distorted profile as the counts of those dups
1451   // will be accumulated by the profile merger.
1452   GlobalValue::LinkageTypes Linkage = GO.getLinkage();
1453   if (Linkage != GlobalValue::ExternalWeakLinkage &&
1454       Linkage != GlobalValue::AvailableExternallyLinkage)
1455     return false;
1456
1457   return true;
1458 }
1459
1460 // Check if INSTR_PROF_RAW_VERSION_VAR is defined.
1461 bool isIRPGOFlagSet(const Module *M) {
1462   const GlobalVariable *IRInstrVar =
1463       M->getNamedGlobal(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
1464   if (!IRInstrVar || IRInstrVar->hasLocalLinkage())
1465     return false;
1466
1467   // For CSPGO+LTO, this variable might be marked as non-prevailing and we only
1468   // have the decl.
1469   if (IRInstrVar->isDeclaration())
1470     return true;
1471
1472   // Check if the flag is set.
1473   if (!IRInstrVar->hasInitializer())
1474     return false;
1475
1476   auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
1477   if (!InitVal)
1478     return false;
1479   return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
1480 }
1481
1482 // Check if we can safely rename this Comdat function.
1483 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
1484   if (F.getName().empty())
1485     return false;
1486   if (!needsComdatForCounter(F, *(F.getParent())))
1487     return false;
1488   // Unsafe to rename the address-taken function (which can be used in
1489   // function comparison).
1490   if (CheckAddressTaken && F.hasAddressTaken())
1491     return false;
1492   // Only safe to do if this function may be discarded if it is not used
1493   // in the compilation unit.
1494   if (!GlobalValue::isDiscardableIfUnused(F.getLinkage()))
1495     return false;
1496
1497   // For AvailableExternallyLinkage functions.
1498   if (!F.hasComdat()) {
1499     assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage);
1500     return true;
1501   }
1502   return true;
1503 }
1504
1505 // Create the variable for the profile file name.
1506 void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
1507   if (InstrProfileOutput.empty())
1508     return;
1509   Constant *ProfileNameConst =
1510       ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
1511   GlobalVariable *ProfileNameVar = new GlobalVariable(
1512       M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
1513       ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
1514   ProfileNameVar->setVisibility(GlobalValue::HiddenVisibility);
1515   Triple TT(M.getTargetTriple());
1516   if (TT.supportsCOMDAT()) {
1517     ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
1518     ProfileNameVar->setComdat(M.getOrInsertComdat(
1519         StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
1520   }
1521 }
1522
1523 Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
1524                                      const std::string &TestFilename,
1525                                      bool IsCS) {
1526   auto GetProfileSum = [IsCS](const std::string &Filename,
1527                               CountSumOrPercent &Sum) -> Error {
1528     // This function is only used from llvm-profdata that doesn't use any kind
1529     // of VFS. Just create a default RealFileSystem to read profiles.
1530     auto FS = vfs::getRealFileSystem();
1531     auto ReaderOrErr = InstrProfReader::create(Filename, *FS);
1532     if (Error E = ReaderOrErr.takeError()) {
1533       return E;
1534     }
1535     auto Reader = std::move(ReaderOrErr.get());
1536     Reader->accumulateCounts(Sum, IsCS);
1537     return Error::success();
1538   };
1539   auto Ret = GetProfileSum(BaseFilename, Base);
1540   if (Ret)
1541     return Ret;
1542   Ret = GetProfileSum(TestFilename, Test);
1543   if (Ret)
1544     return Ret;
1545   this->BaseFilename = &BaseFilename;
1546   this->TestFilename = &TestFilename;
1547   Valid = true;
1548   return Error::success();
1549 }
1550
1551 void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) {
1552   Mismatch.NumEntries += 1;
1553   Mismatch.CountSum += MismatchFunc.CountSum / Test.CountSum;
1554   for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1555     if (Test.ValueCounts[I] >= 1.0f)
1556       Mismatch.ValueCounts[I] +=
1557           MismatchFunc.ValueCounts[I] / Test.ValueCounts[I];
1558   }
1559 }
1560
1561 void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) {
1562   Unique.NumEntries += 1;
1563   Unique.CountSum += UniqueFunc.CountSum / Test.CountSum;
1564   for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1565     if (Test.ValueCounts[I] >= 1.0f)
1566       Unique.ValueCounts[I] += UniqueFunc.ValueCounts[I] / Test.ValueCounts[I];
1567   }
1568 }
1569
1570 void OverlapStats::dump(raw_fd_ostream &OS) const {
1571   if (!Valid)
1572     return;
1573
1574   const char *EntryName =
1575       (Level == ProgramLevel ? "functions" : "edge counters");
1576   if (Level == ProgramLevel) {
1577     OS << "Profile overlap infomation for base_profile: " << *BaseFilename
1578        << " and test_profile: " << *TestFilename << "\nProgram level:\n";
1579   } else {
1580     OS << "Function level:\n"
1581        << "  Function: " << FuncName << " (Hash=" << FuncHash << ")\n";
1582   }
1583
1584   OS << "  # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";
1585   if (Mismatch.NumEntries)
1586     OS << "  # of " << EntryName << " mismatch: " << Mismatch.NumEntries
1587        << "\n";
1588   if (Unique.NumEntries)
1589     OS << "  # of " << EntryName
1590        << " only in test_profile: " << Unique.NumEntries << "\n";
1591
1592   OS << "  Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)
1593      << "\n";
1594   if (Mismatch.NumEntries)
1595     OS << "  Mismatched count percentage (Edge): "
1596        << format("%.3f%%", Mismatch.CountSum * 100) << "\n";
1597   if (Unique.NumEntries)
1598     OS << "  Percentage of Edge profile only in test_profile: "
1599        << format("%.3f%%", Unique.CountSum * 100) << "\n";
1600   OS << "  Edge profile base count sum: " << format("%.0f", Base.CountSum)
1601      << "\n"
1602      << "  Edge profile test count sum: " << format("%.0f", Test.CountSum)
1603      << "\n";
1604
1605   for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {
1606     if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)
1607       continue;
1608     char ProfileKindName[20] = {0};
1609     switch (I) {
1610     case IPVK_IndirectCallTarget:
1611       strncpy(ProfileKindName, "IndirectCall", 19);
1612       break;
1613     case IPVK_MemOPSize:
1614       strncpy(ProfileKindName, "MemOP", 19);
1615       break;
1616     case IPVK_VTableTarget:
1617       strncpy(ProfileKindName, "VTable", 19);
1618       break;
1619     default:
1620       snprintf(ProfileKindName, 19, "VP[%d]", I);
1621       break;
1622     }
1623     OS << "  " << ProfileKindName
1624        << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)
1625        << "\n";
1626     if (Mismatch.NumEntries)
1627       OS << "  Mismatched count percentage (" << ProfileKindName
1628          << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";
1629     if (Unique.NumEntries)
1630       OS << "  Percentage of " << ProfileKindName
1631          << " profile only in test_profile: "
1632          << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";
1633     OS << "  " << ProfileKindName
1634        << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])
1635        << "\n"
1636        << "  " << ProfileKindName
1637        << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])
1638        << "\n";
1639   }
1640 }
1641
1642 namespace IndexedInstrProf {
1643 Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) {
1644   using namespace support;
1645   static_assert(std::is_standard_layout_v<Header>,
1646                 "Use standard layout for Header for simplicity");
1647   Header H;
1648
1649   H.Magic = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1650   // Check the magic number.
1651   if (H.Magic != IndexedInstrProf::Magic)
1652     return make_error<InstrProfError>(instrprof_error::bad_magic);
1653
1654   // Read the version.
1655   H.Version = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1656   if (H.getIndexedProfileVersion() >
1657       IndexedInstrProf::ProfVersion::CurrentVersion)
1658     return make_error<InstrProfError>(instrprof_error::unsupported_version);
1659
1660   static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
1661                 "Please update the reader as needed when a new field is added "
1662                 "or when indexed profile version gets bumped.");
1663
1664   Buffer += sizeof(uint64_t); // Skip Header.Unused field.
1665   H.HashType = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1666   H.HashOffset = endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1667   if (H.getIndexedProfileVersion() >= 8)
1668     H.MemProfOffset =
1669         endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1670   if (H.getIndexedProfileVersion() >= 9)
1671     H.BinaryIdOffset =
1672         endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1673   // Version 11 is handled by this condition.
1674   if (H.getIndexedProfileVersion() >= 10)
1675     H.TemporalProfTracesOffset =
1676         endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1677   if (H.getIndexedProfileVersion() >= 12)
1678     H.VTableNamesOffset =
1679         endian::readNext<uint64_t, llvm::endianness::little>(Buffer);
1680   return H;
1681 }
1682
1683 uint64_t Header::getIndexedProfileVersion() const {
1684   return GET_VERSION(Version);
1685 }
1686
1687 size_t Header::size() const {
1688   switch (getIndexedProfileVersion()) {
1689     // To retain backward compatibility, new fields must be appended to the end
1690     // of the header, and byte offset of existing fields shouldn't change when
1691     // indexed profile version gets incremented.
1692     static_assert(
1693         IndexedInstrProf::ProfVersion::CurrentVersion == Version12,
1694         "Please update the size computation below if a new field has "
1695         "been added to the header; for a version bump without new "
1696         "fields, add a case statement to fall through to the latest version.");
1697   case 12ull:
1698     return 72;
1699   case 11ull:
1700     [[fallthrough]];
1701   case 10ull:
1702     return 64;
1703   case 9ull:
1704     return 56;
1705   case 8ull:
1706     return 48;
1707   default: // Version7 (when the backwards compatible header was introduced).
1708     return 40;
1709   }
1710 }
1711
1712 } // namespace IndexedInstrProf
1713
1714 } // end namespace llvm