llvm/unittests/ProfileData/InstrProfTest.cpp

   1 //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/ADT/STLExtras.h"
  10 #include "llvm/IR/DerivedTypes.h"
  11 #include "llvm/IR/Function.h"
  12 #include "llvm/IR/IRBuilder.h"
  13 #include "llvm/IR/LLVMContext.h"
  14 #include "llvm/IR/Module.h"
  15 #include "llvm/ProfileData/InstrProfReader.h"
  16 #include "llvm/ProfileData/InstrProfWriter.h"
  17 #include "llvm/ProfileData/MemProf.h"
  18 #include "llvm/ProfileData/MemProfData.inc"
  19 #include "llvm/Support/Compression.h"
  20 #include "llvm/Support/raw_ostream.h"
  21 #include "llvm/Testing/Support/Error.h"
  22 #include "gtest/gtest.h"
  23 #include <cstdarg>
  24 #include <initializer_list>
  25 #include <optional>
  26
  27 using namespace llvm;
  28 using ::testing::EndsWith;
  29 using ::testing::IsSubsetOf;
  30 using ::testing::SizeIs;
  31 using ::testing::UnorderedElementsAre;
  32
  33 [[nodiscard]] static ::testing::AssertionResult
  34 ErrorEquals(instrprof_error Expected, Error E) {
  35   instrprof_error Found;
  36   std::string FoundMsg;
  37   handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
  38     Found = IPE.get();
  39     FoundMsg = IPE.message();
  40   });
  41   if (Expected == Found)
  42     return ::testing::AssertionSuccess();
  43   return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";
  44 }
  45
  46 namespace llvm {
  47 bool operator==(const TemporalProfTraceTy &lhs,
  48                 const TemporalProfTraceTy &rhs) {
  49   return lhs.Weight == rhs.Weight &&
  50          lhs.FunctionNameRefs == rhs.FunctionNameRefs;
  51 }
  52 } // end namespace llvm
  53
  54 namespace {
  55
  56 struct InstrProfTest : ::testing::Test {
  57   InstrProfWriter Writer;
  58   std::unique_ptr<IndexedInstrProfReader> Reader;
  59
  60   void SetUp() override { Writer.setOutputSparse(false); }
  61
  62   void readProfile(std::unique_ptr<MemoryBuffer> Profile,
  63                    std::unique_ptr<MemoryBuffer> Remapping = nullptr) {
  64     auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),
  65                                                       std::move(Remapping));
  66     EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());
  67     Reader = std::move(ReaderOrErr.get());
  68   }
  69 };
  70
  71 struct SparseInstrProfTest : public InstrProfTest {
  72   void SetUp() override { Writer.setOutputSparse(true); }
  73 };
  74
  75 struct InstrProfReaderWriterTest
  76     : public InstrProfTest,
  77       public ::testing::WithParamInterface<
  78           std::tuple<bool, uint64_t, llvm::endianness>> {
  79   void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); }
  80   void TearDown() override {
  81     // Reset writer value profile data endianness after each test case. Note
  82     // it's not necessary to reset reader value profile endianness for each test
  83     // case. Each test case creates a new reader; at reader initialization time,
  84     // it uses the endianness from hash table object (which is little by
  85     // default).
  86     Writer.setValueProfDataEndianness(llvm::endianness::little);
  87   }
  88
  89   uint64_t getProfWeight() const { return std::get<1>(GetParam()); }
  90
  91   llvm::endianness getEndianness() const { return std::get<2>(GetParam()); }
  92 };
  93
  94 struct MaybeSparseInstrProfTest : public InstrProfTest,
  95                                   public ::testing::WithParamInterface<bool> {
  96   void SetUp() override { Writer.setOutputSparse(GetParam()); }
  97 };
  98
  99 TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {
 100   auto Profile = Writer.writeBuffer();
 101   readProfile(std::move(Profile));
 102   ASSERT_TRUE(Reader->begin() == Reader->end());
 103 }
 104
 105 static const auto Err = [](Error E) {
 106   consumeError(std::move(E));
 107   FAIL();
 108 };
 109
 110 TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {
 111   Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);
 112   auto Profile = Writer.writeBuffer();
 113   readProfile(std::move(Profile));
 114
 115   auto I = Reader->begin(), E = Reader->end();
 116   ASSERT_TRUE(I != E);
 117   ASSERT_EQ(StringRef("foo"), I->Name);
 118   ASSERT_EQ(0x1234U, I->Hash);
 119   ASSERT_EQ(4U, I->Counts.size());
 120   ASSERT_EQ(1U, I->Counts[0]);
 121   ASSERT_EQ(2U, I->Counts[1]);
 122   ASSERT_EQ(3U, I->Counts[2]);
 123   ASSERT_EQ(4U, I->Counts[3]);
 124   ASSERT_TRUE(++I == E);
 125 }
 126
 127 TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
 128   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
 129   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
 130   auto Profile = Writer.writeBuffer();
 131   readProfile(std::move(Profile));
 132
 133   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("foo", 0x1234);
 134   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 135   ASSERT_EQ(2U, R->Counts.size());
 136   ASSERT_EQ(1U, R->Counts[0]);
 137   ASSERT_EQ(2U, R->Counts[1]);
 138
 139   R = Reader->getInstrProfRecord("foo", 0x1235);
 140   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 141   ASSERT_EQ(2U, R->Counts.size());
 142   ASSERT_EQ(3U, R->Counts[0]);
 143   ASSERT_EQ(4U, R->Counts[1]);
 144
 145   R = Reader->getInstrProfRecord("foo", 0x5678);
 146   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));
 147
 148   R = Reader->getInstrProfRecord("bar", 0x1234);
 149   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));
 150 }
 151
 152 TEST_P(MaybeSparseInstrProfTest, get_function_counts) {
 153   Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);
 154   Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);
 155   auto Profile = Writer.writeBuffer();
 156   readProfile(std::move(Profile));
 157
 158   std::vector<uint64_t> Counts;
 159   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
 160                     Succeeded());
 161   ASSERT_EQ(2U, Counts.size());
 162   ASSERT_EQ(1U, Counts[0]);
 163   ASSERT_EQ(2U, Counts[1]);
 164
 165   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
 166                     Succeeded());
 167   ASSERT_EQ(2U, Counts.size());
 168   ASSERT_EQ(3U, Counts[0]);
 169   ASSERT_EQ(4U, Counts[1]);
 170
 171   Error E1 = Reader->getFunctionCounts("foo", 0x5678, Counts);
 172   ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));
 173
 174   Error E2 = Reader->getFunctionCounts("bar", 0x1234, Counts);
 175   ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));
 176 }
 177
 178 // Profile data is copied from general.proftext
 179 TEST_F(InstrProfTest, get_profile_summary) {
 180   Writer.addRecord({"func1", 0x1234, {97531}}, Err);
 181   Writer.addRecord({"func2", 0x1234, {0, 0}}, Err);
 182   Writer.addRecord(
 183       {"func3",
 184        0x1234,
 185        {2305843009213693952, 1152921504606846976, 576460752303423488,
 186         288230376151711744, 144115188075855872, 72057594037927936}},
 187       Err);
 188   Writer.addRecord({"func4", 0x1234, {0}}, Err);
 189   auto Profile = Writer.writeBuffer();
 190   readProfile(std::move(Profile));
 191
 192   auto VerifySummary = [](ProfileSummary &IPS) mutable {
 193     ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());
 194     ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());
 195     ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());
 196     ASSERT_EQ(10U, IPS.getNumCounts());
 197     ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());
 198     const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();
 199     uint32_t Cutoff = 800000;
 200     auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {
 201       return PE.Cutoff == Cutoff;
 202     };
 203     auto EightyPerc = find_if(Details, Predicate);
 204     Cutoff = 900000;
 205     auto NinetyPerc = find_if(Details, Predicate);
 206     Cutoff = 950000;
 207     auto NinetyFivePerc = find_if(Details, Predicate);
 208     Cutoff = 990000;
 209     auto NinetyNinePerc = find_if(Details, Predicate);
 210     ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);
 211     ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);
 212     ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);
 213     ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);
 214   };
 215   ProfileSummary &PS = Reader->getSummary(/* IsCS */ false);
 216   VerifySummary(PS);
 217
 218   // Test that conversion of summary to and from Metadata works.
 219   LLVMContext Context;
 220   Metadata *MD = PS.getMD(Context);
 221   ASSERT_TRUE(MD);
 222   ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);
 223   ASSERT_TRUE(PSFromMD);
 224   VerifySummary(*PSFromMD);
 225   delete PSFromMD;
 226
 227   // Test that summary can be attached to and read back from module.
 228   Module M("my_module", Context);
 229   M.setProfileSummary(MD, ProfileSummary::PSK_Instr);
 230   MD = M.getProfileSummary(/* IsCS */ false);
 231   ASSERT_TRUE(MD);
 232   PSFromMD = ProfileSummary::getFromMD(MD);
 233   ASSERT_TRUE(PSFromMD);
 234   VerifySummary(*PSFromMD);
 235   delete PSFromMD;
 236 }
 237
 238 TEST_F(InstrProfTest, test_writer_merge) {
 239   Writer.addRecord({"func1", 0x1234, {42}}, Err);
 240
 241   InstrProfWriter Writer2;
 242   Writer2.addRecord({"func2", 0x1234, {0, 0}}, Err);
 243
 244   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
 245
 246   auto Profile = Writer.writeBuffer();
 247   readProfile(std::move(Profile));
 248
 249   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
 250   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 251   ASSERT_EQ(1U, R->Counts.size());
 252   ASSERT_EQ(42U, R->Counts[0]);
 253
 254   R = Reader->getInstrProfRecord("func2", 0x1234);
 255   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 256   ASSERT_EQ(2U, R->Counts.size());
 257   ASSERT_EQ(0U, R->Counts[0]);
 258   ASSERT_EQ(0U, R->Counts[1]);
 259 }
 260
 261 TEST_F(InstrProfTest, test_merge_temporal_prof_traces_truncated) {
 262   uint64_t ReservoirSize = 10;
 263   uint64_t MaxTraceLength = 2;
 264   InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
 265   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
 266                     Succeeded());
 267
 268   TemporalProfTraceTy LargeTrace, SmallTrace;
 269   LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"),
 270                                  IndexedInstrProf::ComputeHash("bar"),
 271                                  IndexedInstrProf::ComputeHash("goo")};
 272   SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"),
 273                                  IndexedInstrProf::ComputeHash("bar")};
 274
 275   SmallVector<TemporalProfTraceTy, 4> Traces = {LargeTrace, SmallTrace};
 276   Writer.addTemporalProfileTraces(Traces, 2);
 277
 278   auto Profile = Writer.writeBuffer();
 279   readProfile(std::move(Profile));
 280
 281   ASSERT_TRUE(Reader->hasTemporalProfile());
 282   EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);
 283   EXPECT_THAT(Reader->getTemporalProfTraces(),
 284               UnorderedElementsAre(SmallTrace, SmallTrace));
 285 }
 286
 287 TEST_F(InstrProfTest, test_merge_traces_from_writer) {
 288   uint64_t ReservoirSize = 10;
 289   uint64_t MaxTraceLength = 10;
 290   InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
 291   InstrProfWriter Writer2(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
 292   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
 293                     Succeeded());
 294   ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile),
 295                     Succeeded());
 296
 297   TemporalProfTraceTy FooTrace, BarTrace;
 298   FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")};
 299   BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")};
 300
 301   SmallVector<TemporalProfTraceTy, 4> Traces1({FooTrace}), Traces2({BarTrace});
 302   Writer.addTemporalProfileTraces(Traces1, 1);
 303   Writer2.addTemporalProfileTraces(Traces2, 1);
 304   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
 305
 306   auto Profile = Writer.writeBuffer();
 307   readProfile(std::move(Profile));
 308
 309   ASSERT_TRUE(Reader->hasTemporalProfile());
 310   EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);
 311   EXPECT_THAT(Reader->getTemporalProfTraces(),
 312               UnorderedElementsAre(FooTrace, BarTrace));
 313 }
 314
 315 TEST_F(InstrProfTest, test_merge_traces_sampled) {
 316   uint64_t ReservoirSize = 3;
 317   uint64_t MaxTraceLength = 10;
 318   InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);
 319   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),
 320                     Succeeded());
 321
 322   TemporalProfTraceTy FooTrace, BarTrace, GooTrace;
 323   FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")};
 324   BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")};
 325   GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("Goo")};
 326
 327   // Add some sampled traces
 328   SmallVector<TemporalProfTraceTy, 4> SampledTraces = {FooTrace, BarTrace,
 329                                                        GooTrace};
 330   Writer.addTemporalProfileTraces(SampledTraces, 5);
 331   // Add some unsampled traces
 332   SmallVector<TemporalProfTraceTy, 4> UnsampledTraces = {BarTrace, GooTrace};
 333   Writer.addTemporalProfileTraces(UnsampledTraces, 2);
 334   UnsampledTraces = {FooTrace};
 335   Writer.addTemporalProfileTraces(UnsampledTraces, 1);
 336
 337   auto Profile = Writer.writeBuffer();
 338   readProfile(std::move(Profile));
 339
 340   ASSERT_TRUE(Reader->hasTemporalProfile());
 341   EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 8U);
 342   // Check that we have a subset of all the traces we added
 343   EXPECT_THAT(Reader->getTemporalProfTraces(), SizeIs(ReservoirSize));
 344   EXPECT_THAT(
 345       Reader->getTemporalProfTraces(),
 346       IsSubsetOf({FooTrace, BarTrace, GooTrace, BarTrace, GooTrace, FooTrace}));
 347 }
 348
 349 using ::llvm::memprof::IndexedMemProfRecord;
 350 using ::llvm::memprof::MemInfoBlock;
 351 using FrameIdMapTy =
 352     llvm::MapVector<::llvm::memprof::FrameId, ::llvm::memprof::Frame>;
 353 using CallStackIdMapTy =
 354     llvm::MapVector<::llvm::memprof::CallStackId,
 355                     ::llvm::SmallVector<::llvm::memprof::FrameId>>;
 356
 357 static FrameIdMapTy getFrameMapping() {
 358   FrameIdMapTy Mapping;
 359   Mapping.insert({0, {0x123, 1, 2, false}});
 360   Mapping.insert({1, {0x345, 3, 4, true}});
 361   Mapping.insert({2, {0x125, 5, 6, false}});
 362   Mapping.insert({3, {0x567, 7, 8, true}});
 363   Mapping.insert({4, {0x124, 5, 6, false}});
 364   Mapping.insert({5, {0x789, 8, 9, true}});
 365   return Mapping;
 366 }
 367
 368 static CallStackIdMapTy getCallStackMapping() {
 369   CallStackIdMapTy Mapping;
 370   Mapping.insert({0x111, {0, 1}});
 371   Mapping.insert({0x222, {2, 3}});
 372   Mapping.insert({0x333, {4, 5}});
 373   return Mapping;
 374 }
 375
 376 // Populate all of the fields of MIB.
 377 MemInfoBlock makeFullMIB() {
 378   MemInfoBlock MIB;
 379 #define MIBEntryDef(NameTag, Name, Type) MIB.NameTag;
 380 #include "llvm/ProfileData/MIBEntryDef.inc"
 381 #undef MIBEntryDef
 382   return MIB;
 383 }
 384
 385 // Populate those fields returned by getHotColdSchema.
 386 MemInfoBlock makePartialMIB() {
 387   MemInfoBlock MIB;
 388   MIB.AllocCount = 1;
 389   MIB.TotalSize = 5;
 390   MIB.TotalLifetime = 10;
 391   MIB.TotalLifetimeAccessDensity = 23;
 392   return MIB;
 393 }
 394
 395 IndexedMemProfRecord makeRecord(
 396     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
 397         AllocFrames,
 398     std::initializer_list<std::initializer_list<::llvm::memprof::FrameId>>
 399         CallSiteFrames,
 400     const MemInfoBlock &Block = makeFullMIB()) {
 401   llvm::memprof::IndexedMemProfRecord MR;
 402   for (const auto &Frames : AllocFrames)
 403     MR.AllocSites.emplace_back(Frames, llvm::memprof::hashCallStack(Frames),
 404                                Block);
 405   for (const auto &Frames : CallSiteFrames)
 406     MR.CallSites.push_back(Frames);
 407   return MR;
 408 }
 409
 410 IndexedMemProfRecord
 411 makeRecordV2(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames,
 412              std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames,
 413              const MemInfoBlock &Block, const memprof::MemProfSchema &Schema) {
 414   llvm::memprof::IndexedMemProfRecord MR;
 415   for (const auto &CSId : AllocFrames)
 416     // We don't populate IndexedAllocationInfo::CallStack because we use it only
 417     // in Version1.
 418     MR.AllocSites.emplace_back(CSId, Block, Schema);
 419   for (const auto &CSId : CallSiteFrames)
 420     MR.CallSiteIds.push_back(CSId);
 421   return MR;
 422 }
 423
 424 MATCHER_P(EqualsRecord, Want, "") {
 425   const memprof::MemProfRecord &Got = arg;
 426
 427   auto PrintAndFail = [&]() {
 428     std::string Buffer;
 429     llvm::raw_string_ostream OS(Buffer);
 430     OS << "Want:\n";
 431     Want.print(OS);
 432     OS << "Got:\n";
 433     Got.print(OS);
 434     *result_listener << "MemProf Record differs!\n" << Buffer;
 435     return false;
 436   };
 437
 438   if (Want.AllocSites.size() != Got.AllocSites.size())
 439     return PrintAndFail();
 440   if (Want.CallSites.size() != Got.CallSites.size())
 441     return PrintAndFail();
 442
 443   for (size_t I = 0; I < Got.AllocSites.size(); I++) {
 444     if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)
 445       return PrintAndFail();
 446     if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)
 447       return PrintAndFail();
 448   }
 449
 450   for (size_t I = 0; I < Got.CallSites.size(); I++) {
 451     if (Want.CallSites[I] != Got.CallSites[I])
 452       return PrintAndFail();
 453   }
 454   return true;
 455 }
 456
 457 TEST_F(InstrProfTest, test_memprof_v0) {
 458   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
 459                     Succeeded());
 460
 461   const IndexedMemProfRecord IndexedMR = makeRecord(
 462       /*AllocFrames=*/
 463       {
 464           {0, 1},
 465           {2, 3},
 466       },
 467       /*CallSiteFrames=*/{
 468           {4, 5},
 469       });
 470
 471   memprof::IndexedMemProfData MemProfData;
 472   MemProfData.Frames = getFrameMapping();
 473   MemProfData.Records.try_emplace(0x9999, IndexedMR);
 474   Writer.addMemProfData(MemProfData, Err);
 475
 476   auto Profile = Writer.writeBuffer();
 477   readProfile(std::move(Profile));
 478
 479   auto RecordOr = Reader->getMemProfRecord(0x9999);
 480   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
 481   const memprof::MemProfRecord &Record = RecordOr.get();
 482
 483   std::optional<memprof::FrameId> LastUnmappedFrameId;
 484   auto IdToFrameCallback = [&](const memprof::FrameId Id) {
 485     auto Iter = MemProfData.Frames.find(Id);
 486     if (Iter == MemProfData.Frames.end()) {
 487       LastUnmappedFrameId = Id;
 488       return memprof::Frame(0, 0, 0, false);
 489     }
 490     return Iter->second;
 491   };
 492
 493   const memprof::MemProfRecord WantRecord(IndexedMR, IdToFrameCallback);
 494   ASSERT_EQ(LastUnmappedFrameId, std::nullopt)
 495       << "could not map frame id: " << *LastUnmappedFrameId;
 496   EXPECT_THAT(WantRecord, EqualsRecord(Record));
 497 }
 498
 499 TEST_F(InstrProfTest, test_memprof_v2_full_schema) {
 500   const MemInfoBlock MIB = makeFullMIB();
 501
 502   Writer.setMemProfVersionRequested(memprof::Version2);
 503   Writer.setMemProfFullSchema(true);
 504
 505   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
 506                     Succeeded());
 507
 508   const IndexedMemProfRecord IndexedMR = makeRecordV2(
 509       /*AllocFrames=*/{0x111, 0x222},
 510       /*CallSiteFrames=*/{0x333}, MIB, memprof::getFullSchema());
 511   memprof::IndexedMemProfData MemProfData;
 512   MemProfData.Frames = getFrameMapping();
 513   MemProfData.CallStacks = getCallStackMapping();
 514   MemProfData.Records.try_emplace(0x9999, IndexedMR);
 515   Writer.addMemProfData(MemProfData, Err);
 516
 517   auto Profile = Writer.writeBuffer();
 518   readProfile(std::move(Profile));
 519
 520   auto RecordOr = Reader->getMemProfRecord(0x9999);
 521   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
 522   const memprof::MemProfRecord &Record = RecordOr.get();
 523
 524   memprof::FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
 525       MemProfData.Frames);
 526   memprof::CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
 527       MemProfData.CallStacks, FrameIdConv);
 528
 529   const ::llvm::memprof::MemProfRecord WantRecord =
 530       IndexedMR.toMemProfRecord(CSIdConv);
 531   ASSERT_EQ(FrameIdConv.LastUnmappedId, std::nullopt)
 532       << "could not map frame id: " << *FrameIdConv.LastUnmappedId;
 533   ASSERT_EQ(CSIdConv.LastUnmappedId, std::nullopt)
 534       << "could not map call stack id: " << *CSIdConv.LastUnmappedId;
 535   EXPECT_THAT(WantRecord, EqualsRecord(Record));
 536 }
 537
 538 TEST_F(InstrProfTest, test_memprof_v2_partial_schema) {
 539   const MemInfoBlock MIB = makePartialMIB();
 540
 541   Writer.setMemProfVersionRequested(memprof::Version2);
 542   Writer.setMemProfFullSchema(false);
 543
 544   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
 545                     Succeeded());
 546
 547   const IndexedMemProfRecord IndexedMR = makeRecordV2(
 548       /*AllocFrames=*/{0x111, 0x222},
 549       /*CallSiteFrames=*/{0x333}, MIB, memprof::getHotColdSchema());
 550   memprof::IndexedMemProfData MemProfData;
 551   MemProfData.Frames = getFrameMapping();
 552   MemProfData.CallStacks = getCallStackMapping();
 553   MemProfData.Records.try_emplace(0x9999, IndexedMR);
 554   Writer.addMemProfData(MemProfData, Err);
 555
 556   auto Profile = Writer.writeBuffer();
 557   readProfile(std::move(Profile));
 558
 559   auto RecordOr = Reader->getMemProfRecord(0x9999);
 560   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
 561   const memprof::MemProfRecord &Record = RecordOr.get();
 562
 563   memprof::FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
 564       MemProfData.Frames);
 565   memprof::CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
 566       MemProfData.CallStacks, FrameIdConv);
 567
 568   const ::llvm::memprof::MemProfRecord WantRecord =
 569       IndexedMR.toMemProfRecord(CSIdConv);
 570   ASSERT_EQ(FrameIdConv.LastUnmappedId, std::nullopt)
 571       << "could not map frame id: " << *FrameIdConv.LastUnmappedId;
 572   ASSERT_EQ(CSIdConv.LastUnmappedId, std::nullopt)
 573       << "could not map call stack id: " << *CSIdConv.LastUnmappedId;
 574   EXPECT_THAT(WantRecord, EqualsRecord(Record));
 575 }
 576
 577 TEST_F(InstrProfTest, test_caller_callee_pairs) {
 578   const MemInfoBlock MIB = makePartialMIB();
 579
 580   Writer.setMemProfVersionRequested(memprof::Version3);
 581   Writer.setMemProfFullSchema(false);
 582
 583   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
 584                     Succeeded());
 585
 586   // Call Hierarchy
 587   //
 588   // Function GUID:0x123
 589   //   Line: 1, Column: 2
 590   //     Function GUID: 0x234
 591   //       Line: 3, Column: 4
 592   //         new(...)
 593   //   Line: 5, Column: 6
 594   //     Function GUID: 0x345
 595   //       Line: 7, Column: 8
 596   //         new(...)
 597
 598   const IndexedMemProfRecord IndexedMR = makeRecordV2(
 599       /*AllocFrames=*/{0x111, 0x222},
 600       /*CallSiteFrames=*/{}, MIB, memprof::getHotColdSchema());
 601
 602   memprof::IndexedMemProfData MemProfData;
 603   MemProfData.Frames.try_emplace(0, 0x123, 1, 2, false);
 604   MemProfData.Frames.try_emplace(1, 0x234, 3, 4, true);
 605   MemProfData.Frames.try_emplace(2, 0x123, 5, 6, false);
 606   MemProfData.Frames.try_emplace(3, 0x345, 7, 8, true);
 607   MemProfData.CallStacks.try_emplace(
 608       0x111, std::initializer_list<memprof::FrameId>{1, 0});
 609   MemProfData.CallStacks.try_emplace(
 610       0x222, std::initializer_list<memprof::FrameId>{3, 2});
 611   MemProfData.Records.try_emplace(0x9999, IndexedMR);
 612   Writer.addMemProfData(MemProfData, Err);
 613
 614   auto Profile = Writer.writeBuffer();
 615   readProfile(std::move(Profile));
 616
 617   auto Pairs = Reader->getMemProfCallerCalleePairs();
 618   ASSERT_THAT(Pairs, SizeIs(3));
 619
 620   auto It = Pairs.find(0x123);
 621   ASSERT_NE(It, Pairs.end());
 622   ASSERT_THAT(It->second, SizeIs(2));
 623   EXPECT_THAT(It->second[0], testing::Pair(testing::FieldsAre(1U, 2U), 0x234U));
 624   EXPECT_THAT(It->second[1], testing::Pair(testing::FieldsAre(5U, 6U), 0x345U));
 625
 626   It = Pairs.find(0x234);
 627   ASSERT_NE(It, Pairs.end());
 628   ASSERT_THAT(It->second, SizeIs(1));
 629   EXPECT_THAT(It->second[0], testing::Pair(testing::FieldsAre(3U, 4U), 0U));
 630
 631   It = Pairs.find(0x345);
 632   ASSERT_NE(It, Pairs.end());
 633   ASSERT_THAT(It->second, SizeIs(1));
 634   EXPECT_THAT(It->second[0], testing::Pair(testing::FieldsAre(7U, 8U), 0U));
 635 }
 636
 637 TEST_F(InstrProfTest, test_memprof_getrecord_error) {
 638   ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),
 639                     Succeeded());
 640
 641   const IndexedMemProfRecord IndexedMR = makeRecord(
 642       /*AllocFrames=*/
 643       {
 644           {0, 1},
 645           {2, 3},
 646       },
 647       /*CallSiteFrames=*/{
 648           {4, 5},
 649       });
 650   // We skip adding the frame mappings here unlike the test_memprof unit test
 651   // above to exercise the failure path when getMemProfRecord is invoked.
 652   Writer.addMemProfRecord(/*Id=*/0x9999, IndexedMR);
 653
 654   auto Profile = Writer.writeBuffer();
 655   readProfile(std::move(Profile));
 656
 657   // Missing frames give a hash_mismatch error.
 658   auto RecordOr = Reader->getMemProfRecord(0x9999);
 659   ASSERT_TRUE(
 660       ErrorEquals(instrprof_error::hash_mismatch, RecordOr.takeError()));
 661
 662   // Missing functions give a unknown_function error.
 663   RecordOr = Reader->getMemProfRecord(0x1111);
 664   ASSERT_TRUE(
 665       ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError()));
 666 }
 667
 668 TEST_F(InstrProfTest, test_memprof_merge) {
 669   Writer.addRecord({"func1", 0x1234, {42}}, Err);
 670
 671   InstrProfWriter Writer2;
 672   Writer2.setMemProfVersionRequested(memprof::Version3);
 673   ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),
 674                     Succeeded());
 675
 676   const IndexedMemProfRecord IndexedMR = makeRecordV2(
 677       /*AllocFrames=*/{0x111, 0x222},
 678       /*CallSiteFrames=*/{}, makePartialMIB(), memprof::getHotColdSchema());
 679
 680   memprof::IndexedMemProfData MemProfData;
 681   MemProfData.Frames = getFrameMapping();
 682   MemProfData.CallStacks = getCallStackMapping();
 683   MemProfData.Records.try_emplace(0x9999, IndexedMR);
 684   Writer2.addMemProfData(MemProfData, Err);
 685
 686   ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),
 687                     Succeeded());
 688   Writer.mergeRecordsFromWriter(std::move(Writer2), Err);
 689
 690   Writer.setMemProfVersionRequested(memprof::Version3);
 691   auto Profile = Writer.writeBuffer();
 692   readProfile(std::move(Profile));
 693
 694   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
 695   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 696   ASSERT_EQ(1U, R->Counts.size());
 697   ASSERT_EQ(42U, R->Counts[0]);
 698
 699   auto RecordOr = Reader->getMemProfRecord(0x9999);
 700   ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());
 701   const memprof::MemProfRecord &Record = RecordOr.get();
 702
 703   std::optional<memprof::FrameId> LastUnmappedFrameId;
 704
 705   memprof::FrameIdConverter<decltype(MemProfData.Frames)> FrameIdConv(
 706       MemProfData.Frames);
 707   memprof::CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
 708       MemProfData.CallStacks, FrameIdConv);
 709
 710   const ::llvm::memprof::MemProfRecord WantRecord =
 711       IndexedMR.toMemProfRecord(CSIdConv);
 712   ASSERT_EQ(LastUnmappedFrameId, std::nullopt)
 713       << "could not map frame id: " << *LastUnmappedFrameId;
 714   EXPECT_THAT(WantRecord, EqualsRecord(Record));
 715 }
 716
 717 TEST_F(InstrProfTest, test_irpgo_function_name) {
 718   LLVMContext Ctx;
 719   auto M = std::make_unique<Module>("MyModule.cpp", Ctx);
 720   auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);
 721
 722   std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;
 723   Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo");
 724   Data.emplace_back("InternalFoo", Function::InternalLinkage,
 725                     "MyModule.cpp;InternalFoo");
 726   Data.emplace_back("\01-[C dynamicFoo:]", Function::ExternalLinkage,
 727                     "-[C dynamicFoo:]");
 728   Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage,
 729                     "MyModule.cpp;-[C internalFoo:]");
 730
 731   for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data)
 732     Function::Create(FTy, Linkage, Name, M.get());
 733
 734   for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) {
 735     auto *F = M->getFunction(Name);
 736     auto IRPGOFuncName = getIRPGOFuncName(*F);
 737     EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName);
 738
 739     auto [Filename, ParsedIRPGOFuncName] = getParsedIRPGOName(IRPGOFuncName);
 740     StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName;
 741     if (ExpectedParsedIRPGOFuncName.consume_front("MyModule.cpp;")) {
 742       EXPECT_EQ(Filename, "MyModule.cpp");
 743     } else {
 744       EXPECT_EQ(Filename, "");
 745     }
 746     EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName);
 747   }
 748 }
 749
 750 TEST_F(InstrProfTest, test_pgo_function_name) {
 751   LLVMContext Ctx;
 752   auto M = std::make_unique<Module>("MyModule.cpp", Ctx);
 753   auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);
 754
 755   std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;
 756   Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo");
 757   Data.emplace_back("InternalFoo", Function::InternalLinkage,
 758                     "MyModule.cpp:InternalFoo");
 759   Data.emplace_back("\01-[C externalFoo:]", Function::ExternalLinkage,
 760                     "-[C externalFoo:]");
 761   Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage,
 762                     "MyModule.cpp:-[C internalFoo:]");
 763
 764   for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data)
 765     Function::Create(FTy, Linkage, Name, M.get());
 766
 767   for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) {
 768     auto *F = M->getFunction(Name);
 769     EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName);
 770   }
 771 }
 772
 773 TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) {
 774   LLVMContext Ctx;
 775   auto M = std::make_unique<Module>("MyModule.cpp", Ctx);
 776   auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);
 777   auto *InternalFooF =
 778       Function::Create(FTy, Function::InternalLinkage, "InternalFoo", M.get());
 779   auto *ExternalFooF =
 780       Function::Create(FTy, Function::ExternalLinkage, "ExternalFoo", M.get());
 781
 782   auto *InternalBarF =
 783       Function::Create(FTy, Function::InternalLinkage, "InternalBar", M.get());
 784   auto *ExternalBarF =
 785       Function::Create(FTy, Function::ExternalLinkage, "ExternalBar", M.get());
 786
 787   Writer.addRecord({getIRPGOFuncName(*InternalFooF), 0x1234, {1}}, Err);
 788   Writer.addRecord({getIRPGOFuncName(*ExternalFooF), 0x5678, {1}}, Err);
 789   // Write a record with a deprecated name
 790   Writer.addRecord({getPGOFuncName(*InternalBarF), 0x1111, {2}}, Err);
 791   Writer.addRecord({getPGOFuncName(*ExternalBarF), 0x2222, {2}}, Err);
 792
 793   auto Profile = Writer.writeBuffer();
 794   readProfile(std::move(Profile));
 795
 796   EXPECT_THAT_EXPECTED(
 797       Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234,
 798                                  getPGOFuncName(*InternalFooF)),
 799       Succeeded());
 800   EXPECT_THAT_EXPECTED(
 801       Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678,
 802                                  getPGOFuncName(*ExternalFooF)),
 803       Succeeded());
 804   // Ensure we can still read this old record name
 805   EXPECT_THAT_EXPECTED(
 806       Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111,
 807                                  getPGOFuncName(*InternalBarF)),
 808       Succeeded());
 809   EXPECT_THAT_EXPECTED(
 810       Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222,
 811                                  getPGOFuncName(*ExternalBarF)),
 812       Succeeded());
 813 }
 814
 815 // callee1 to callee6 are from vtable1 to vtable6 respectively.
 816 static const char callee1[] = "callee1";
 817 static const char callee2[] = "callee2";
 818 static const char callee3[] = "callee3";
 819 static const char callee4[] = "callee4";
 820 static const char callee5[] = "callee5";
 821 static const char callee6[] = "callee6";
 822 // callee7 and callee8 are not from any vtables.
 823 static const char callee7[] = "callee7";
 824 static const char callee8[] = "callee8";
 825 // 'callee' is primarily used to create multiple-element vtables.
 826 static const char callee[] = "callee";
 827 static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)};
 828 static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)};
 829 static const uint64_t vtable3[] = {
 830     uint64_t(callee),
 831     uint64_t(callee3),
 832 };
 833 static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)};
 834 static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)};
 835 static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)};
 836
 837 // Returns the address of callee with a numbered suffix in vtable.
 838 static uint64_t getCalleeAddress(const uint64_t *vtableAddr) {
 839   uint64_t CalleeAddr;
 840   // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3,
 841   // and the 1st element in the rest of vtables.
 842   if (vtableAddr == vtable1 || vtableAddr == vtable3)
 843     CalleeAddr = uint64_t(vtableAddr) + 8;
 844   else
 845     CalleeAddr = uint64_t(vtableAddr);
 846   return CalleeAddr;
 847 }
 848
 849 TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) {
 850   NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});
 851
 852   // 4 indirect call value sites.
 853   {
 854     Record1.reserveSites(IPVK_IndirectCallTarget, 4);
 855     InstrProfValueData VD0[] = {
 856         {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};
 857     Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);
 858     // No value profile data at the second site.
 859     Record1.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);
 860     InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};
 861     Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);
 862     InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}};
 863     Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);
 864   }
 865
 866   // 2 vtable value sites.
 867   {
 868     InstrProfValueData VD0[] = {
 869         {getCalleeAddress(vtable1), 1},
 870         {getCalleeAddress(vtable2), 2},
 871         {getCalleeAddress(vtable3), 3},
 872     };
 873     InstrProfValueData VD2[] = {
 874         {getCalleeAddress(vtable1), 1},
 875         {getCalleeAddress(vtable2), 2},
 876     };
 877     Record1.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);
 878     Record1.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);
 879   }
 880
 881   Writer.addRecord(std::move(Record1), getProfWeight(), Err);
 882   Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);
 883   Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);
 884   Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);
 885   Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err);
 886   Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err);
 887
 888   // Set writer value prof data endianness.
 889   Writer.setValueProfDataEndianness(getEndianness());
 890
 891   auto Profile = Writer.writeBuffer();
 892   readProfile(std::move(Profile));
 893
 894   // Set reader value prof data endianness.
 895   Reader->setValueProfDataEndianness(getEndianness());
 896
 897   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
 898   ASSERT_THAT_ERROR(R.takeError(), Succeeded());
 899
 900   // Test the number of instrumented indirect call sites and the number of
 901   // profiled values at each site.
 902   ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));
 903
 904   // Test the number of instrumented vtable sites and the number of profiled
 905   // values at each site.
 906   ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U);
 907
 908   // First indirect site.
 909   {
 910     auto VD = R->getValueArrayForSite(IPVK_IndirectCallTarget, 0);
 911     ASSERT_THAT(VD, SizeIs(3));
 912
 913     EXPECT_EQ(VD[0].Count, 3U * getProfWeight());
 914     EXPECT_EQ(VD[1].Count, 2U * getProfWeight());
 915     EXPECT_EQ(VD[2].Count, 1U * getProfWeight());
 916
 917     EXPECT_STREQ((const char *)VD[0].Value, "callee3");
 918     EXPECT_STREQ((const char *)VD[1].Value, "callee2");
 919     EXPECT_STREQ((const char *)VD[2].Value, "callee1");
 920   }
 921
 922   EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 1), SizeIs(0));
 923   EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 2), SizeIs(2));
 924   EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 3), SizeIs(2));
 925
 926   // First vtable site.
 927   {
 928     auto VD = R->getValueArrayForSite(IPVK_VTableTarget, 0);
 929     ASSERT_THAT(VD, SizeIs(3));
 930
 931     EXPECT_EQ(VD[0].Count, 3U * getProfWeight());
 932     EXPECT_EQ(VD[1].Count, 2U * getProfWeight());
 933     EXPECT_EQ(VD[2].Count, 1U * getProfWeight());
 934
 935     EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3));
 936     EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2));
 937     EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1));
 938   }
 939
 940   // Second vtable site.
 941   {
 942     auto VD = R->getValueArrayForSite(IPVK_VTableTarget, 1);
 943     ASSERT_THAT(VD, SizeIs(2));
 944
 945     EXPECT_EQ(VD[0].Count, 2U * getProfWeight());
 946     EXPECT_EQ(VD[1].Count, 1U * getProfWeight());
 947
 948     EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2));
 949     EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1));
 950   }
 951 }
 952
 953 INSTANTIATE_TEST_SUITE_P(
 954     WeightAndEndiannessTest, InstrProfReaderWriterTest,
 955     ::testing::Combine(
 956         ::testing::Bool(),          /* Sparse */
 957         ::testing::Values(1U, 10U), /* ProfWeight */
 958         ::testing::Values(llvm::endianness::big,
 959                           llvm::endianness::little) /* Endianness */
 960         ));
 961
 962 TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
 963   NamedInstrProfRecord Record("caller", 0x1234, {1, 2});
 964   Record.reserveSites(IPVK_IndirectCallTarget, 1);
 965   InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},
 966                               {4000, 4}, {6000, 6}};
 967   Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);
 968   Writer.addRecord(std::move(Record), Err);
 969   auto Profile = Writer.writeBuffer();
 970   readProfile(std::move(Profile));
 971   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
 972   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 973
 974   LLVMContext Ctx;
 975   std::unique_ptr<Module> M(new Module("MyModule", Ctx));
 976   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
 977                                         /*isVarArg=*/false);
 978   Function *F =
 979       Function::Create(FTy, Function::ExternalLinkage, "caller", M.get());
 980   BasicBlock *BB = BasicBlock::Create(Ctx, "", F);
 981
 982   IRBuilder<> Builder(BB);
 983   BasicBlock *TBB = BasicBlock::Create(Ctx, "", F);
 984   BasicBlock *FBB = BasicBlock::Create(Ctx, "", F);
 985
 986   // Use branch instruction to annotate with value profile data for simplicity
 987   Instruction *Inst = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
 988   Instruction *Inst2 = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);
 989   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0);
 990
 991   uint64_t T;
 992   auto ValueData =
 993       getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);
 994   ASSERT_THAT(ValueData, SizeIs(3));
 995   ASSERT_EQ(21U, T);
 996   // The result should be sorted already:
 997   ASSERT_EQ(6000U, ValueData[0].Value);
 998   ASSERT_EQ(6U, ValueData[0].Count);
 999   ASSERT_EQ(5000U, ValueData[1].Value);
1000   ASSERT_EQ(5U, ValueData[1].Count);
1001   ASSERT_EQ(4000U, ValueData[2].Value);
1002   ASSERT_EQ(4U, ValueData[2].Count);
1003   ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 1, T);
1004   ASSERT_THAT(ValueData, SizeIs(1));
1005   ASSERT_EQ(21U, T);
1006
1007   ValueData = getValueProfDataFromInst(*Inst2, IPVK_IndirectCallTarget, 5, T);
1008   ASSERT_THAT(ValueData, SizeIs(0));
1009
1010   // Remove the MD_prof metadata
1011   Inst->setMetadata(LLVMContext::MD_prof, 0);
1012   // Annotate 5 records this time.
1013   annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);
1014   ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);
1015   ASSERT_THAT(ValueData, SizeIs(5));
1016   ASSERT_EQ(21U, T);
1017   ASSERT_EQ(6000U, ValueData[0].Value);
1018   ASSERT_EQ(6U, ValueData[0].Count);
1019   ASSERT_EQ(5000U, ValueData[1].Value);
1020   ASSERT_EQ(5U, ValueData[1].Count);
1021   ASSERT_EQ(4000U, ValueData[2].Value);
1022   ASSERT_EQ(4U, ValueData[2].Count);
1023   ASSERT_EQ(3000U, ValueData[3].Value);
1024   ASSERT_EQ(3U, ValueData[3].Count);
1025   ASSERT_EQ(2000U, ValueData[4].Value);
1026   ASSERT_EQ(2U, ValueData[4].Count);
1027
1028   // Remove the MD_prof metadata
1029   Inst->setMetadata(LLVMContext::MD_prof, 0);
1030   // Annotate with 4 records.
1031   InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},
1032                               {5000, 2}, {6000, 1}};
1033   annotateValueSite(*M, *Inst, ArrayRef(VD0Sorted).slice(2), 10,
1034                     IPVK_IndirectCallTarget, 5);
1035   ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);
1036   ASSERT_THAT(ValueData, SizeIs(4));
1037   ASSERT_EQ(10U, T);
1038   ASSERT_EQ(3000U, ValueData[0].Value);
1039   ASSERT_EQ(4U, ValueData[0].Count);
1040   ASSERT_EQ(4000U, ValueData[1].Value);
1041   ASSERT_EQ(3U, ValueData[1].Count);
1042   ASSERT_EQ(5000U, ValueData[2].Value);
1043   ASSERT_EQ(2U, ValueData[2].Count);
1044   ASSERT_EQ(6000U, ValueData[3].Value);
1045   ASSERT_EQ(1U, ValueData[3].Count);
1046 }
1047
1048 TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) {
1049   static const char caller[] = "caller";
1050   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
1051   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
1052
1053   // 5 value sites for indirect calls.
1054   {
1055     Record11.reserveSites(IPVK_IndirectCallTarget, 5);
1056     InstrProfValueData VD0[] = {{uint64_t(callee1), 1},
1057                                 {uint64_t(callee2), 2},
1058                                 {uint64_t(callee3), 3},
1059                                 {uint64_t(callee4), 4}};
1060     Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);
1061
1062     // No value profile data at the second site.
1063     Record11.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);
1064
1065     InstrProfValueData VD2[] = {
1066         {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
1067     Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);
1068
1069     InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}};
1070     Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);
1071
1072     InstrProfValueData VD4[] = {
1073         {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
1074     Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, nullptr);
1075   }
1076   // 3 value sites for vtables.
1077   {
1078     Record11.reserveSites(IPVK_VTableTarget, 3);
1079     InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1},
1080                                 {getCalleeAddress(vtable2), 2},
1081                                 {getCalleeAddress(vtable3), 3},
1082                                 {getCalleeAddress(vtable4), 4}};
1083     Record11.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);
1084
1085     InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1},
1086                                 {getCalleeAddress(vtable2), 2},
1087                                 {getCalleeAddress(vtable3), 3}};
1088     Record11.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);
1089
1090     InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},
1091                                 {getCalleeAddress(vtable2), 2},
1092                                 {getCalleeAddress(vtable3), 3}};
1093     Record11.addValueData(IPVK_VTableTarget, 2, VD4, nullptr);
1094   }
1095
1096   // A different record for the same caller.
1097   Record12.reserveSites(IPVK_IndirectCallTarget, 5);
1098   InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};
1099   Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, nullptr);
1100
1101   // No value profile data at the second site.
1102   Record12.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);
1103
1104   InstrProfValueData VD22[] = {
1105       {uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};
1106   Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, nullptr);
1107
1108   Record12.addValueData(IPVK_IndirectCallTarget, 3, {}, nullptr);
1109
1110   InstrProfValueData VD42[] = {
1111       {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};
1112   Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, nullptr);
1113
1114   // 3 value sites for vtables.
1115   {
1116     Record12.reserveSites(IPVK_VTableTarget, 3);
1117     InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5},
1118                                 {getCalleeAddress(vtable3), 3}};
1119     Record12.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);
1120
1121     InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1},
1122                                 {getCalleeAddress(vtable3), 3},
1123                                 {getCalleeAddress(vtable4), 4}};
1124     Record12.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);
1125
1126     InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},
1127                                 {getCalleeAddress(vtable2), 2},
1128                                 {getCalleeAddress(vtable3), 3}};
1129     Record12.addValueData(IPVK_VTableTarget, 2, VD4, nullptr);
1130   }
1131
1132   Writer.addRecord(std::move(Record11), Err);
1133   // Merge profile data.
1134   Writer.addRecord(std::move(Record12), Err);
1135
1136   Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);
1137   Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);
1138   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
1139   Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);
1140   Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);
1141   Writer.addRecord({callee7, 0x1235, {3, 5}}, Err);
1142   Writer.addRecord({callee8, 0x1235, {3, 5}}, Err);
1143   auto Profile = Writer.writeBuffer();
1144   readProfile(std::move(Profile));
1145
1146   // Test the number of instrumented value sites and the number of profiled
1147   // values for each site.
1148   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
1149   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
1150   // For indirect calls.
1151   ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
1152   // For vtables.
1153   ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U);
1154
1155   // Test the merged values for indirect calls.
1156   {
1157     auto VD = R->getValueArrayForSite(IPVK_IndirectCallTarget, 0);
1158     ASSERT_THAT(VD, SizeIs(4));
1159     EXPECT_STREQ((const char *)VD[0].Value, "callee2");
1160     EXPECT_EQ(VD[0].Count, 7U);
1161     EXPECT_STREQ((const char *)VD[1].Value, "callee3");
1162     EXPECT_EQ(VD[1].Count, 6U);
1163     EXPECT_STREQ((const char *)VD[2].Value, "callee4");
1164     EXPECT_EQ(VD[2].Count, 4U);
1165     EXPECT_STREQ((const char *)VD[3].Value, "callee1");
1166     EXPECT_EQ(VD[3].Count, 1U);
1167
1168     ASSERT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 1), SizeIs(0));
1169
1170     auto VD_2 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 2);
1171     ASSERT_THAT(VD_2, SizeIs(4));
1172     EXPECT_STREQ((const char *)VD_2[0].Value, "callee3");
1173     EXPECT_EQ(VD_2[0].Count, 6U);
1174     EXPECT_STREQ((const char *)VD_2[1].Value, "callee4");
1175     EXPECT_EQ(VD_2[1].Count, 4U);
1176     EXPECT_STREQ((const char *)VD_2[2].Value, "callee2");
1177     EXPECT_EQ(VD_2[2].Count, 3U);
1178     EXPECT_STREQ((const char *)VD_2[3].Value, "callee1");
1179     EXPECT_EQ(VD_2[3].Count, 1U);
1180
1181     auto VD_3 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 3);
1182     ASSERT_THAT(VD_3, SizeIs(2));
1183     EXPECT_STREQ((const char *)VD_3[0].Value, "callee8");
1184     EXPECT_EQ(VD_3[0].Count, 2U);
1185     EXPECT_STREQ((const char *)VD_3[1].Value, "callee7");
1186     EXPECT_EQ(VD_3[1].Count, 1U);
1187
1188     auto VD_4 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 4);
1189     ASSERT_THAT(VD_4, SizeIs(3));
1190     EXPECT_STREQ((const char *)VD_4[0].Value, "callee3");
1191     EXPECT_EQ(VD_4[0].Count, 6U);
1192     EXPECT_STREQ((const char *)VD_4[1].Value, "callee2");
1193     EXPECT_EQ(VD_4[1].Count, 4U);
1194     EXPECT_STREQ((const char *)VD_4[2].Value, "callee1");
1195     EXPECT_EQ(VD_4[2].Count, 2U);
1196   }
1197
1198   // Test the merged values for vtables
1199   {
1200     auto VD0 = R->getValueArrayForSite(IPVK_VTableTarget, 0);
1201     ASSERT_THAT(VD0, SizeIs(4));
1202     EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
1203     EXPECT_EQ(VD0[0].Count, 7U);
1204     EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
1205     EXPECT_EQ(VD0[1].Count, 6U);
1206     EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4));
1207     EXPECT_EQ(VD0[2].Count, 4U);
1208     EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1));
1209     EXPECT_EQ(VD0[3].Count, 1U);
1210
1211     auto VD1 = R->getValueArrayForSite(IPVK_VTableTarget, 1);
1212     ASSERT_THAT(VD1, SizeIs(4));
1213     EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3));
1214     EXPECT_EQ(VD1[0].Count, 6U);
1215     EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4));
1216     EXPECT_EQ(VD1[1].Count, 4U);
1217     EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2));
1218     EXPECT_EQ(VD1[2].Count, 3U);
1219     EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1));
1220     EXPECT_EQ(VD1[3].Count, 1U);
1221
1222     auto VD2 = R->getValueArrayForSite(IPVK_VTableTarget, 2);
1223     ASSERT_THAT(VD2, SizeIs(3));
1224     EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3));
1225     EXPECT_EQ(VD2[0].Count, 6U);
1226     EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2));
1227     EXPECT_EQ(VD2[1].Count, 4U);
1228     EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1));
1229     EXPECT_EQ(VD2[2].Count, 2U);
1230   }
1231 }
1232
1233 struct ValueProfileMergeEdgeCaseTest
1234     : public InstrProfTest,
1235       public ::testing::WithParamInterface<std::tuple<bool, uint32_t>> {
1236   void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); }
1237
1238   uint32_t getValueProfileKind() const { return std::get<1>(GetParam()); }
1239 };
1240
1241 TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_saturation) {
1242   const uint32_t ValueKind = getValueProfileKind();
1243   static const char bar[] = "bar";
1244   const uint64_t ProfiledValue = 0x5678;
1245
1246   const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max();
1247   const uint64_t MaxEdgeCount = getInstrMaxCountValue();
1248
1249   instrprof_error Result;
1250   auto Err = [&](Error E) {
1251     Result = std::get<0>(InstrProfError::take(std::move(E)));
1252   };
1253   Result = instrprof_error::success;
1254   Writer.addRecord({"foo", 0x1234, {1}}, Err);
1255   ASSERT_EQ(Result, instrprof_error::success);
1256
1257   // Verify counter overflow.
1258   Result = instrprof_error::success;
1259   Writer.addRecord({"foo", 0x1234, {MaxEdgeCount}}, Err);
1260   ASSERT_EQ(Result, instrprof_error::counter_overflow);
1261
1262   Result = instrprof_error::success;
1263   Writer.addRecord({bar, 0x9012, {8}}, Err);
1264   ASSERT_EQ(Result, instrprof_error::success);
1265
1266   NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});
1267   Record4.reserveSites(ValueKind, 1);
1268   InstrProfValueData VD4[] = {{ProfiledValue, 1}};
1269   Record4.addValueData(ValueKind, 0, VD4, nullptr);
1270   Result = instrprof_error::success;
1271   Writer.addRecord(std::move(Record4), Err);
1272   ASSERT_EQ(Result, instrprof_error::success);
1273
1274   // Verify value data counter overflow.
1275   NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});
1276   Record5.reserveSites(ValueKind, 1);
1277   InstrProfValueData VD5[] = {{ProfiledValue, MaxValCount}};
1278   Record5.addValueData(ValueKind, 0, VD5, nullptr);
1279   Result = instrprof_error::success;
1280   Writer.addRecord(std::move(Record5), Err);
1281   ASSERT_EQ(Result, instrprof_error::counter_overflow);
1282
1283   auto Profile = Writer.writeBuffer();
1284   readProfile(std::move(Profile));
1285
1286   // Verify saturation of counts.
1287   Expected<InstrProfRecord> ReadRecord1 =
1288       Reader->getInstrProfRecord("foo", 0x1234);
1289   ASSERT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
1290   EXPECT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]);
1291
1292   Expected<InstrProfRecord> ReadRecord2 =
1293       Reader->getInstrProfRecord("baz", 0x5678);
1294   ASSERT_TRUE(bool(ReadRecord2));
1295   ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind));
1296   auto VD = ReadRecord2->getValueArrayForSite(ValueKind, 0);
1297   EXPECT_EQ(ProfiledValue, VD[0].Value);
1298   EXPECT_EQ(MaxValCount, VD[0].Count);
1299 }
1300
1301 // This test tests that when there are too many values for a given site, the
1302 // merged results are properly truncated.
1303 TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_site_trunc) {
1304   const uint32_t ValueKind = getValueProfileKind();
1305   static const char caller[] = "caller";
1306
1307   NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});
1308   NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});
1309
1310   // 2 value sites.
1311   Record11.reserveSites(ValueKind, 2);
1312   InstrProfValueData VD0[255];
1313   for (int I = 0; I < 255; I++) {
1314     VD0[I].Value = 2 * I;
1315     VD0[I].Count = 2 * I + 1000;
1316   }
1317
1318   Record11.addValueData(ValueKind, 0, VD0, nullptr);
1319   Record11.addValueData(ValueKind, 1, {}, nullptr);
1320
1321   Record12.reserveSites(ValueKind, 2);
1322   InstrProfValueData VD1[255];
1323   for (int I = 0; I < 255; I++) {
1324     VD1[I].Value = 2 * I + 1;
1325     VD1[I].Count = 2 * I + 1001;
1326   }
1327
1328   Record12.addValueData(ValueKind, 0, VD1, nullptr);
1329   Record12.addValueData(ValueKind, 1, {}, nullptr);
1330
1331   Writer.addRecord(std::move(Record11), Err);
1332   // Merge profile data.
1333   Writer.addRecord(std::move(Record12), Err);
1334
1335   auto Profile = Writer.writeBuffer();
1336   readProfile(std::move(Profile));
1337
1338   Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
1339   ASSERT_THAT_ERROR(R.takeError(), Succeeded());
1340   ASSERT_EQ(2U, R->getNumValueSites(ValueKind));
1341   auto VD = R->getValueArrayForSite(ValueKind, 0);
1342   EXPECT_THAT(VD, SizeIs(255));
1343   for (unsigned I = 0; I < 255; I++) {
1344     EXPECT_EQ(VD[I].Value, 509 - I);
1345     EXPECT_EQ(VD[I].Count, 1509 - I);
1346   }
1347 }
1348
1349 INSTANTIATE_TEST_SUITE_P(
1350     EdgeCaseTest, ValueProfileMergeEdgeCaseTest,
1351     ::testing::Combine(::testing::Bool(), /* Sparse */
1352                        ::testing::Values(IPVK_IndirectCallTarget,
1353                                          IPVK_MemOPSize,
1354                                          IPVK_VTableTarget) /* ValueKind */
1355                        ));
1356
1357 static void addValueProfData(InstrProfRecord &Record) {
1358   // Add test data for indirect calls.
1359   {
1360     Record.reserveSites(IPVK_IndirectCallTarget, 6);
1361     InstrProfValueData VD0[] = {{uint64_t(callee1), 400},
1362                                 {uint64_t(callee2), 1000},
1363                                 {uint64_t(callee3), 500},
1364                                 {uint64_t(callee4), 300},
1365                                 {uint64_t(callee5), 100}};
1366     Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);
1367     InstrProfValueData VD1[] = {{uint64_t(callee5), 800},
1368                                 {uint64_t(callee3), 1000},
1369                                 {uint64_t(callee2), 2500},
1370                                 {uint64_t(callee1), 1300}};
1371     Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, nullptr);
1372     InstrProfValueData VD2[] = {{uint64_t(callee6), 800},
1373                                 {uint64_t(callee3), 1000},
1374                                 {uint64_t(callee4), 5500}};
1375     Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);
1376     InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},
1377                                 {uint64_t(callee3), 2000}};
1378     Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);
1379     Record.addValueData(IPVK_IndirectCallTarget, 4, {}, nullptr);
1380     InstrProfValueData VD5[] = {{uint64_t(callee7), 1234},
1381                                 {uint64_t(callee8), 5678}};
1382     Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, nullptr);
1383   }
1384
1385   // Add test data for vtables
1386   {
1387     Record.reserveSites(IPVK_VTableTarget, 4);
1388     InstrProfValueData VD0[] = {
1389         {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000},
1390         {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300},
1391         {getCalleeAddress(vtable5), 100},
1392     };
1393     InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800},
1394                                 {getCalleeAddress(vtable3), 1000},
1395                                 {getCalleeAddress(vtable2), 2500},
1396                                 {getCalleeAddress(vtable1), 1300}};
1397     InstrProfValueData VD2[] = {
1398         {getCalleeAddress(vtable6), 800},
1399         {getCalleeAddress(vtable3), 1000},
1400         {getCalleeAddress(vtable4), 5500},
1401     };
1402     InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800},
1403                                 {getCalleeAddress(vtable3), 2000}};
1404     Record.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);
1405     Record.addValueData(IPVK_VTableTarget, 1, VD1, nullptr);
1406     Record.addValueData(IPVK_VTableTarget, 2, VD2, nullptr);
1407     Record.addValueData(IPVK_VTableTarget, 3, VD3, nullptr);
1408   }
1409 }
1410
1411 TEST(ValueProfileReadWriteTest, value_prof_data_read_write) {
1412   InstrProfRecord SrcRecord({1ULL << 31, 2});
1413   addValueProfData(SrcRecord);
1414   std::unique_ptr<ValueProfData> VPData =
1415       ValueProfData::serializeFrom(SrcRecord);
1416
1417   InstrProfRecord Record({1ULL << 31, 2});
1418   VPData->deserializeTo(Record, nullptr);
1419
1420   // Now read data from Record and sanity check the data
1421   ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget));
1422
1423   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
1424     return VD1.Count > VD2.Count;
1425   };
1426
1427   SmallVector<InstrProfValueData> VD_0(
1428       Record.getValueArrayForSite(IPVK_IndirectCallTarget, 0));
1429   ASSERT_THAT(VD_0, SizeIs(5));
1430   llvm::sort(VD_0, Cmp);
1431   EXPECT_STREQ((const char *)VD_0[0].Value, "callee2");
1432   EXPECT_EQ(1000U, VD_0[0].Count);
1433   EXPECT_STREQ((const char *)VD_0[1].Value, "callee3");
1434   EXPECT_EQ(500U, VD_0[1].Count);
1435   EXPECT_STREQ((const char *)VD_0[2].Value, "callee1");
1436   EXPECT_EQ(400U, VD_0[2].Count);
1437   EXPECT_STREQ((const char *)VD_0[3].Value, "callee4");
1438   EXPECT_EQ(300U, VD_0[3].Count);
1439   EXPECT_STREQ((const char *)VD_0[4].Value, "callee5");
1440   EXPECT_EQ(100U, VD_0[4].Count);
1441
1442   SmallVector<InstrProfValueData> VD_1(
1443       Record.getValueArrayForSite(IPVK_IndirectCallTarget, 1));
1444   ASSERT_THAT(VD_1, SizeIs(4));
1445   llvm::sort(VD_1, Cmp);
1446   EXPECT_STREQ((const char *)VD_1[0].Value, "callee2");
1447   EXPECT_EQ(VD_1[0].Count, 2500U);
1448   EXPECT_STREQ((const char *)VD_1[1].Value, "callee1");
1449   EXPECT_EQ(VD_1[1].Count, 1300U);
1450   EXPECT_STREQ((const char *)VD_1[2].Value, "callee3");
1451   EXPECT_EQ(VD_1[2].Count, 1000U);
1452   EXPECT_STREQ((const char *)VD_1[3].Value, "callee5");
1453   EXPECT_EQ(VD_1[3].Count, 800U);
1454
1455   SmallVector<InstrProfValueData> VD_2(
1456       Record.getValueArrayForSite(IPVK_IndirectCallTarget, 2));
1457   ASSERT_THAT(VD_2, SizeIs(3));
1458   llvm::sort(VD_2, Cmp);
1459   EXPECT_STREQ((const char *)VD_2[0].Value, "callee4");
1460   EXPECT_EQ(VD_2[0].Count, 5500U);
1461   EXPECT_STREQ((const char *)VD_2[1].Value, "callee3");
1462   EXPECT_EQ(VD_2[1].Count, 1000U);
1463   EXPECT_STREQ((const char *)VD_2[2].Value, "callee6");
1464   EXPECT_EQ(VD_2[2].Count, 800U);
1465
1466   SmallVector<InstrProfValueData> VD_3(
1467       Record.getValueArrayForSite(IPVK_IndirectCallTarget, 3));
1468   ASSERT_THAT(VD_3, SizeIs(2));
1469   llvm::sort(VD_3, Cmp);
1470   EXPECT_STREQ((const char *)VD_3[0].Value, "callee3");
1471   EXPECT_EQ(VD_3[0].Count, 2000U);
1472   EXPECT_STREQ((const char *)VD_3[1].Value, "callee2");
1473   EXPECT_EQ(VD_3[1].Count, 1800U);
1474
1475   ASSERT_THAT(Record.getValueArrayForSite(IPVK_IndirectCallTarget, 4),
1476               SizeIs(0));
1477   ASSERT_THAT(Record.getValueArrayForSite(IPVK_IndirectCallTarget, 5),
1478               SizeIs(2));
1479
1480   ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);
1481
1482   SmallVector<InstrProfValueData> VD0(
1483       Record.getValueArrayForSite(IPVK_VTableTarget, 0));
1484   ASSERT_THAT(VD0, SizeIs(5));
1485   llvm::sort(VD0, Cmp);
1486   EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));
1487   EXPECT_EQ(VD0[0].Count, 1000U);
1488   EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));
1489   EXPECT_EQ(VD0[1].Count, 500U);
1490   EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1));
1491   EXPECT_EQ(VD0[2].Count, 400U);
1492   EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4));
1493   EXPECT_EQ(VD0[3].Count, 300U);
1494   EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5));
1495   EXPECT_EQ(VD0[4].Count, 100U);
1496
1497   SmallVector<InstrProfValueData> VD1(
1498       Record.getValueArrayForSite(IPVK_VTableTarget, 1));
1499   ASSERT_THAT(VD1, SizeIs(4));
1500   llvm::sort(VD1, Cmp);
1501   EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2));
1502   EXPECT_EQ(VD1[0].Count, 2500U);
1503   EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1));
1504   EXPECT_EQ(VD1[1].Count, 1300U);
1505   EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3));
1506   EXPECT_EQ(VD1[2].Count, 1000U);
1507   EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5));
1508   EXPECT_EQ(VD1[3].Count, 800U);
1509
1510   SmallVector<InstrProfValueData> VD2(
1511       Record.getValueArrayForSite(IPVK_VTableTarget, 2));
1512   ASSERT_THAT(VD2, SizeIs(3));
1513   llvm::sort(VD2, Cmp);
1514   EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4));
1515   EXPECT_EQ(VD2[0].Count, 5500U);
1516   EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3));
1517   EXPECT_EQ(VD2[1].Count, 1000U);
1518   EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6));
1519   EXPECT_EQ(VD2[2].Count, 800U);
1520
1521   SmallVector<InstrProfValueData> VD3(
1522       Record.getValueArrayForSite(IPVK_VTableTarget, 3));
1523   ASSERT_THAT(VD3, SizeIs(2));
1524   llvm::sort(VD3, Cmp);
1525   EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3));
1526   EXPECT_EQ(VD3[0].Count, 2000U);
1527   EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2));
1528   EXPECT_EQ(VD3[1].Count, 1800U);
1529 }
1530
1531 TEST(ValueProfileReadWriteTest, symtab_mapping) {
1532   NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});
1533   addValueProfData(SrcRecord);
1534   std::unique_ptr<ValueProfData> VPData =
1535       ValueProfData::serializeFrom(SrcRecord);
1536
1537   NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});
1538   InstrProfSymtab Symtab;
1539   Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);
1540   Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);
1541   Symtab.mapAddress(uint64_t(callee3), 0x3000ULL);
1542   Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);
1543   // Missing mapping for callee5
1544
1545   auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t {
1546     return uint64_t(vtable);
1547   };
1548   auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t {
1549     return uint64_t(vtable) + 16;
1550   };
1551   auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t {
1552     return uint64_t(vtable) + 8;
1553   };
1554   // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not
1555   // mapped.
1556   Symtab.mapVTableAddress(getVTableStartAddr(vtable1),
1557                           getVTableEndAddr(vtable1), MD5Hash("vtable1"));
1558   Symtab.mapVTableAddress(getVTableStartAddr(vtable2),
1559                           getVTableEndAddr(vtable2), MD5Hash("vtable2"));
1560   Symtab.mapVTableAddress(getVTableStartAddr(vtable3),
1561                           getVTableEndAddr(vtable3), MD5Hash("vtable3"));
1562   Symtab.mapVTableAddress(getVTableStartAddr(vtable4),
1563                           getVTableEndAddr(vtable4), MD5Hash("vtable4"));
1564
1565   VPData->deserializeTo(Record, &Symtab);
1566
1567   // Now read data from Record and sanity check the data
1568   ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U);
1569
1570   // Look up the value correpsonding to the middle of a vtable in symtab and
1571   // test that it's the hash of the name.
1572   EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)),
1573             MD5Hash("vtable1"));
1574   EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)),
1575             MD5Hash("vtable2"));
1576   EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)),
1577             MD5Hash("vtable3"));
1578   EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)),
1579             MD5Hash("vtable4"));
1580
1581   auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
1582     return VD1.Count > VD2.Count;
1583   };
1584   SmallVector<InstrProfValueData> VD_0(
1585       Record.getValueArrayForSite(IPVK_IndirectCallTarget, 0));
1586   ASSERT_THAT(VD_0, SizeIs(5));
1587   llvm::sort(VD_0, Cmp);
1588   ASSERT_EQ(VD_0[0].Value, 0x2000ULL);
1589   ASSERT_EQ(VD_0[0].Count, 1000U);
1590   ASSERT_EQ(VD_0[1].Value, 0x3000ULL);
1591   ASSERT_EQ(VD_0[1].Count, 500U);
1592   ASSERT_EQ(VD_0[2].Value, 0x1000ULL);
1593   ASSERT_EQ(VD_0[2].Count, 400U);
1594
1595   // callee5 does not have a mapped value -- default to 0.
1596   ASSERT_EQ(VD_0[4].Value, 0ULL);
1597
1598   // Sanity check the vtable value data
1599   ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);
1600
1601   {
1602     // The first vtable site.
1603     SmallVector<InstrProfValueData> VD(
1604         Record.getValueArrayForSite(IPVK_VTableTarget, 0));
1605     ASSERT_THAT(VD, SizeIs(5));
1606     llvm::sort(VD, Cmp);
1607     EXPECT_EQ(VD[0].Count, 1000U);
1608     EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
1609     EXPECT_EQ(VD[1].Count, 500U);
1610     EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
1611     EXPECT_EQ(VD[2].Value, MD5Hash("vtable1"));
1612     EXPECT_EQ(VD[2].Count, 400U);
1613     EXPECT_EQ(VD[3].Value, MD5Hash("vtable4"));
1614     EXPECT_EQ(VD[3].Count, 300U);
1615
1616     // vtable5 isn't mapped -- default to 0.
1617     EXPECT_EQ(VD[4].Value, 0U);
1618     EXPECT_EQ(VD[4].Count, 100U);
1619   }
1620
1621   {
1622     // The second vtable site.
1623     SmallVector<InstrProfValueData> VD(
1624         Record.getValueArrayForSite(IPVK_VTableTarget, 1));
1625     ASSERT_THAT(VD, SizeIs(4));
1626     llvm::sort(VD, Cmp);
1627     EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));
1628     EXPECT_EQ(VD[0].Count, 2500U);
1629     EXPECT_EQ(VD[1].Value, MD5Hash("vtable1"));
1630     EXPECT_EQ(VD[1].Count, 1300U);
1631
1632     EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));
1633     EXPECT_EQ(VD[2].Count, 1000U);
1634     // vtable5 isn't mapped -- default to 0.
1635     EXPECT_EQ(VD[3].Value, 0U);
1636     EXPECT_EQ(VD[3].Count, 800U);
1637   }
1638
1639   {
1640     // The third vtable site.
1641     SmallVector<InstrProfValueData> VD(
1642         Record.getValueArrayForSite(IPVK_VTableTarget, 2));
1643     ASSERT_THAT(VD, SizeIs(3));
1644     llvm::sort(VD, Cmp);
1645     EXPECT_EQ(VD[0].Count, 5500U);
1646     EXPECT_EQ(VD[0].Value, MD5Hash("vtable4"));
1647     EXPECT_EQ(VD[1].Count, 1000U);
1648     EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));
1649     // vtable6 isn't mapped -- default to 0.
1650     EXPECT_EQ(VD[2].Value, 0U);
1651     EXPECT_EQ(VD[2].Count, 800U);
1652   }
1653
1654   {
1655     // The fourth vtable site.
1656     SmallVector<InstrProfValueData> VD(
1657         Record.getValueArrayForSite(IPVK_VTableTarget, 3));
1658     ASSERT_THAT(VD, SizeIs(2));
1659     llvm::sort(VD, Cmp);
1660     EXPECT_EQ(VD[0].Count, 2000U);
1661     EXPECT_EQ(VD[0].Value, MD5Hash("vtable3"));
1662     EXPECT_EQ(VD[1].Count, 1800U);
1663     EXPECT_EQ(VD[1].Value, MD5Hash("vtable2"));
1664   }
1665 }
1666
1667 TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {
1668   Writer.addRecord({"foo", 0x1234, {1ULL << 31, 2}}, Err);
1669   Writer.addRecord({"bar", 0, {1ULL << 63}}, Err);
1670   Writer.addRecord({"baz", 0x5678, {0, 0, 0, 0}}, Err);
1671   auto Profile = Writer.writeBuffer();
1672   readProfile(std::move(Profile));
1673
1674   ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));
1675 }
1676
1677 TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {
1678   Writer.addRecord({"foo", 0x1234, {1, 2}}, 3, Err);
1679   Writer.addRecord({"foo", 0x1235, {3, 4}}, 5, Err);
1680   auto Profile = Writer.writeBuffer();
1681   readProfile(std::move(Profile));
1682
1683   std::vector<uint64_t> Counts;
1684   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),
1685                     Succeeded());
1686   ASSERT_EQ(2U, Counts.size());
1687   ASSERT_EQ(3U, Counts[0]);
1688   ASSERT_EQ(6U, Counts[1]);
1689
1690   EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),
1691                     Succeeded());
1692   ASSERT_EQ(2U, Counts.size());
1693   ASSERT_EQ(15U, Counts[0]);
1694   ASSERT_EQ(20U, Counts[1]);
1695 }
1696
1697 // Testing symtab creator interface used by indexed profile reader.
1698 TEST(SymtabTest, instr_prof_symtab_test) {
1699   std::vector<StringRef> FuncNames;
1700   FuncNames.push_back("func1");
1701   FuncNames.push_back("func2");
1702   FuncNames.push_back("func3");
1703   FuncNames.push_back("bar1");
1704   FuncNames.push_back("bar2");
1705   FuncNames.push_back("bar3");
1706   InstrProfSymtab Symtab;
1707   EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());
1708   StringRef R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func1"));
1709   ASSERT_EQ(StringRef("func1"), R);
1710   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func2"));
1711   ASSERT_EQ(StringRef("func2"), R);
1712   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func3"));
1713   ASSERT_EQ(StringRef("func3"), R);
1714   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar1"));
1715   ASSERT_EQ(StringRef("bar1"), R);
1716   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar2"));
1717   ASSERT_EQ(StringRef("bar2"), R);
1718   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar3"));
1719   ASSERT_EQ(StringRef("bar3"), R);
1720
1721   // negative tests
1722   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar4"));
1723   ASSERT_EQ(StringRef(), R);
1724   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("foo4"));
1725   ASSERT_EQ(StringRef(), R);
1726
1727   // Now incrementally update the symtab
1728   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());
1729   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());
1730   EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());
1731
1732   // Check again
1733   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_1"));
1734   ASSERT_EQ(StringRef("blah_1"), R);
1735   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_2"));
1736   ASSERT_EQ(StringRef("blah_2"), R);
1737   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_3"));
1738   ASSERT_EQ(StringRef("blah_3"), R);
1739   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func1"));
1740   ASSERT_EQ(StringRef("func1"), R);
1741   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func2"));
1742   ASSERT_EQ(StringRef("func2"), R);
1743   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func3"));
1744   ASSERT_EQ(StringRef("func3"), R);
1745   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar1"));
1746   ASSERT_EQ(StringRef("bar1"), R);
1747   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar2"));
1748   ASSERT_EQ(StringRef("bar2"), R);
1749   R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar3"));
1750   ASSERT_EQ(StringRef("bar3"), R);
1751 }
1752
1753 // Test that we get an error when creating a bogus symtab.
1754 TEST(SymtabTest, instr_prof_bogus_symtab_empty_func_name) {
1755   InstrProfSymtab Symtab;
1756   EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));
1757 }
1758
1759 // Testing symtab creator interface used by value profile transformer.
1760 TEST(SymtabTest, instr_prof_symtab_module_test) {
1761   LLVMContext Ctx;
1762   std::unique_ptr<Module> M = std::make_unique<Module>("MyModule.cpp", Ctx);
1763   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),
1764                                         /*isVarArg=*/false);
1765   Function::Create(FTy, Function::ExternalLinkage, "Gfoo", M.get());
1766   Function::Create(FTy, Function::ExternalLinkage, "Gblah", M.get());
1767   Function::Create(FTy, Function::ExternalLinkage, "Gbar", M.get());
1768   Function::Create(FTy, Function::InternalLinkage, "Ifoo", M.get());
1769   Function::Create(FTy, Function::InternalLinkage, "Iblah", M.get());
1770   Function::Create(FTy, Function::InternalLinkage, "Ibar", M.get());
1771   Function::Create(FTy, Function::PrivateLinkage, "Pfoo", M.get());
1772   Function::Create(FTy, Function::PrivateLinkage, "Pblah", M.get());
1773   Function::Create(FTy, Function::PrivateLinkage, "Pbar", M.get());
1774   Function::Create(FTy, Function::WeakODRLinkage, "Wfoo", M.get());
1775   Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());
1776   Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());
1777
1778   // [ptr, ptr, ptr]
1779   ArrayType *VTableArrayType = ArrayType::get(
1780       PointerType::get(Ctx, M->getDataLayout().getDefaultGlobalsAddressSpace()),
1781       3);
1782   Constant *Int32TyNull =
1783       llvm::ConstantExpr::getNullValue(PointerType::getUnqual(Ctx));
1784   SmallVector<llvm::Type *, 1> tys = {VTableArrayType};
1785   StructType *VTableType = llvm::StructType::get(Ctx, tys);
1786
1787   // Create two vtables in the module, one with external linkage and the other
1788   // with local linkage.
1789   for (auto [Name, Linkage] :
1790        {std::pair{"ExternalGV", GlobalValue::ExternalLinkage},
1791         {"LocalGV", GlobalValue::InternalLinkage}}) {
1792     llvm::Twine FuncName(Name, StringRef("VFunc"));
1793     Function *VFunc = Function::Create(FTy, Linkage, FuncName, M.get());
1794     GlobalVariable *GV = new llvm::GlobalVariable(
1795         *M, VTableType, /* isConstant= */ true, Linkage,
1796         llvm::ConstantStruct::get(
1797             VTableType,
1798             {llvm::ConstantArray::get(VTableArrayType,
1799                                       {Int32TyNull, Int32TyNull, VFunc})}),
1800         Name);
1801     // Add type metadata for the test data, since vtables with type metadata
1802     // are added to symtab.
1803     GV->addTypeMetadata(16, MDString::get(Ctx, Name));
1804   }
1805
1806   InstrProfSymtab ProfSymtab;
1807   EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());
1808
1809   StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",
1810                        "Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};
1811
1812   for (unsigned I = 0; I < std::size(Funcs); I++) {
1813     Function *F = M->getFunction(Funcs[I]);
1814
1815     std::string IRPGOName = getIRPGOFuncName(*F);
1816     auto IRPGOFuncName =
1817         ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(IRPGOName));
1818     EXPECT_EQ(IRPGOName, IRPGOFuncName);
1819     EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second);
1820     // Ensure we can still read this old record name.
1821     std::string PGOName = getPGOFuncName(*F);
1822     auto PGOFuncName =
1823         ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(PGOName));
1824     EXPECT_EQ(PGOName, PGOFuncName);
1825     EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str()));
1826   }
1827
1828   for (auto [VTableName, PGOName] : {std::pair{"ExternalGV", "ExternalGV"},
1829                                      {"LocalGV", "MyModule.cpp;LocalGV"}}) {
1830     GlobalVariable *GV =
1831         M->getGlobalVariable(VTableName, /* AllowInternal=*/true);
1832
1833     // Test that ProfSymtab returns the expected name given a hash.
1834     std::string IRPGOName = getPGOName(*GV);
1835     EXPECT_STREQ(IRPGOName.c_str(), PGOName);
1836     uint64_t GUID = IndexedInstrProf::ComputeHash(IRPGOName);
1837     EXPECT_EQ(IRPGOName, ProfSymtab.getFuncOrVarName(GUID));
1838     EXPECT_EQ(VTableName, getParsedIRPGOName(IRPGOName).second);
1839
1840     // Test that ProfSymtab returns the expected global variable
1841     EXPECT_EQ(GV, ProfSymtab.getGlobalVariable(GUID));
1842   }
1843 }
1844
1845 // Testing symtab serialization and creator/deserialization interface
1846 // used by coverage map reader, and raw profile reader.
1847 TEST(SymtabTest, instr_prof_symtab_compression_test) {
1848   std::vector<std::string> FuncNames1;
1849   std::vector<std::string> FuncNames2;
1850   for (int I = 0; I < 3; I++) {
1851     std::string str;
1852     raw_string_ostream OS(str);
1853     OS << "func_" << I;
1854     FuncNames1.push_back(OS.str());
1855     str.clear();
1856     OS << "f oooooooooooooo_" << I;
1857     FuncNames1.push_back(OS.str());
1858     str.clear();
1859     OS << "BAR_" << I;
1860     FuncNames2.push_back(OS.str());
1861     str.clear();
1862     OS << "BlahblahBlahblahBar_" << I;
1863     FuncNames2.push_back(OS.str());
1864   }
1865
1866   for (bool DoCompression : {false, true}) {
1867     // Compressing:
1868     std::string FuncNameStrings1;
1869     EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(
1870                           FuncNames1,
1871                           (DoCompression && compression::zlib::isAvailable()),
1872                           FuncNameStrings1),
1873                       Succeeded());
1874
1875     // Compressing:
1876     std::string FuncNameStrings2;
1877     EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(
1878                           FuncNames2,
1879                           (DoCompression && compression::zlib::isAvailable()),
1880                           FuncNameStrings2),
1881                       Succeeded());
1882
1883     for (int Padding = 0; Padding < 2; Padding++) {
1884       // Join with paddings :
1885       std::string FuncNameStrings = FuncNameStrings1;
1886       for (int P = 0; P < Padding; P++) {
1887         FuncNameStrings.push_back('\0');
1888       }
1889       FuncNameStrings += FuncNameStrings2;
1890
1891       // Now decompress:
1892       InstrProfSymtab Symtab;
1893       EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());
1894
1895       // Now do the checks:
1896       // First sampling some data points:
1897       StringRef R =
1898           Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(FuncNames1[0]));
1899       ASSERT_EQ(StringRef("func_0"), R);
1900       R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(FuncNames1[1]));
1901       ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);
1902       for (int I = 0; I < 3; I++) {
1903         std::string N[4];
1904         N[0] = FuncNames1[2 * I];
1905         N[1] = FuncNames1[2 * I + 1];
1906         N[2] = FuncNames2[2 * I];
1907         N[3] = FuncNames2[2 * I + 1];
1908         for (int J = 0; J < 4; J++) {
1909           StringRef R =
1910               Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(N[J]));
1911           ASSERT_EQ(StringRef(N[J]), R);
1912         }
1913       }
1914     }
1915   }
1916 }
1917
1918 TEST_P(MaybeSparseInstrProfTest, remapping_test) {
1919   Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);
1920   Writer.addRecord({"file;_Z3barf", 0x567, {5, 6, 7}}, Err);
1921   auto Profile = Writer.writeBuffer();
1922   readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(
1923     type i l
1924     name 3bar 4quux
1925   )"));
1926
1927   std::vector<uint64_t> Counts;
1928   for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {
1929     EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),
1930                       Succeeded());
1931     ASSERT_EQ(4u, Counts.size());
1932     EXPECT_EQ(1u, Counts[0]);
1933     EXPECT_EQ(2u, Counts[1]);
1934     EXPECT_EQ(3u, Counts[2]);
1935     EXPECT_EQ(4u, Counts[3]);
1936   }
1937
1938   for (StringRef BarName : {"file;_Z3barf", "file;_Z4quuxf"}) {
1939     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),
1940                       Succeeded());
1941     ASSERT_EQ(3u, Counts.size());
1942     EXPECT_EQ(5u, Counts[0]);
1943     EXPECT_EQ(6u, Counts[1]);
1944     EXPECT_EQ(7u, Counts[2]);
1945   }
1946
1947   for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",
1948                             "_Z3barf", "otherfile:_Z4quuxf"}) {
1949     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),
1950                       Failed());
1951     EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),
1952                       Failed());
1953   }
1954 }
1955
1956 TEST_F(SparseInstrProfTest, preserve_no_records) {
1957   Writer.addRecord({"foo", 0x1234, {0}}, Err);
1958   Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);
1959   Writer.addRecord({"baz", 0x4321, {0, 0, 0}}, Err);
1960
1961   auto Profile = Writer.writeBuffer();
1962   readProfile(std::move(Profile));
1963
1964   auto I = Reader->begin(), E = Reader->end();
1965   ASSERT_TRUE(I == E);
1966 }
1967
1968 INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,
1969                          ::testing::Bool());
1970
1971 #if defined(_LP64) && defined(EXPENSIVE_CHECKS)
1972 TEST(ProfileReaderTest, ReadsLargeFiles) {
1973   const size_t LargeSize = 1ULL << 32; // 4GB
1974
1975   auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1976   if (!RawProfile)
1977     GTEST_SKIP();
1978   auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));
1979   ASSERT_TRUE(
1980       std::get<0>(InstrProfError::take(RawProfileReaderOrErr.takeError())) ==
1981       instrprof_error::unrecognized_format);
1982
1983   auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);
1984   if (!IndexedProfile)
1985     GTEST_SKIP();
1986   auto IndexedReaderOrErr =
1987       IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);
1988   ASSERT_TRUE(
1989       std::get<0>(InstrProfError::take(IndexedReaderOrErr.takeError())) ==
1990       instrprof_error::bad_magic);
1991 }
1992 #endif
1993
1994 } // end anonymous namespace