llvm/unittests/Analysis/MemoryProfileInfoTest.cpp

   1 //===- MemoryProfileInfoTest.cpp - Memory Profile Info Unit Tests-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/MemoryProfileInfo.h"
  10 #include "llvm/AsmParser/Parser.h"
  11 #include "llvm/IR/Instructions.h"
  12 #include "llvm/IR/LLVMContext.h"
  13 #include "llvm/IR/Module.h"
  14 #include "llvm/IR/ModuleSummaryIndex.h"
  15 #include "llvm/Support/CommandLine.h"
  16 #include "llvm/Support/SourceMgr.h"
  17 #include "gtest/gtest.h"
  18 #include <cstring>
  19 #include <sys/types.h>
  20
  21 using namespace llvm;
  22 using namespace llvm::memprof;
  23
  24 extern cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
  25 extern cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
  26 extern cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold;
  27
  28 namespace {
  29
  30 class MemoryProfileInfoTest : public testing::Test {
  31 protected:
  32   std::unique_ptr<Module> makeLLVMModule(LLVMContext &C, const char *IR) {
  33     SMDiagnostic Err;
  34     std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
  35     if (!Mod)
  36       Err.print("MemoryProfileInfoTest", errs());
  37     return Mod;
  38   }
  39
  40   std::unique_ptr<ModuleSummaryIndex> makeLLVMIndex(const char *Summary) {
  41     SMDiagnostic Err;
  42     std::unique_ptr<ModuleSummaryIndex> Index =
  43         parseSummaryIndexAssemblyString(Summary, Err);
  44     if (!Index)
  45       Err.print("MemoryProfileInfoTest", errs());
  46     return Index;
  47   }
  48
  49   // This looks for a call that has the given value name, which
  50   // is the name of the value being assigned the call return value.
  51   CallBase *findCall(Function &F, const char *Name = nullptr) {
  52     for (auto &BB : F)
  53       for (auto &I : BB)
  54         if (auto *CB = dyn_cast<CallBase>(&I))
  55           if (!Name || CB->getName() == Name)
  56             return CB;
  57     return nullptr;
  58   }
  59 };
  60
  61 // Test getAllocType helper.
  62 // Basic checks on the allocation type for values just above and below
  63 // the thresholds.
  64 TEST_F(MemoryProfileInfoTest, GetAllocType) {
  65   const uint64_t AllocCount = 2;
  66   // To be cold we require that
  67   // ((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
  68   //    MemProfLifetimeAccessDensityColdThreshold
  69   // so compute the ColdTotalLifetimeAccessDensityThreshold at the threshold.
  70   const uint64_t ColdTotalLifetimeAccessDensityThreshold =
  71       (uint64_t)(MemProfLifetimeAccessDensityColdThreshold * AllocCount * 100);
  72   // To be cold we require that
  73   // ((float)TotalLifetime) / AllocCount >=
  74   //    MemProfAveLifetimeColdThreshold * 1000
  75   // so compute the TotalLifetime right at the threshold.
  76   const uint64_t ColdTotalLifetimeThreshold =
  77       MemProfAveLifetimeColdThreshold * AllocCount * 1000;
  78   // To be hot we require that
  79   // ((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
  80   //    MemProfMinAveLifetimeAccessDensityHotThreshold
  81   // so compute the HotTotalLifetimeAccessDensityThreshold  at the threshold.
  82   const uint64_t HotTotalLifetimeAccessDensityThreshold =
  83       (uint64_t)(MemProfMinAveLifetimeAccessDensityHotThreshold * AllocCount * 100);
  84
  85
  86   // Test Hot
  87   // More accesses per byte per sec than hot threshold is hot.
  88   EXPECT_EQ(getAllocType(HotTotalLifetimeAccessDensityThreshold + 1, AllocCount,
  89                          ColdTotalLifetimeThreshold + 1),
  90             AllocationType::Hot);
  91
  92   // Test Cold
  93   // Long lived with less accesses per byte per sec than cold threshold is cold.
  94   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold - 1, AllocCount,
  95                          ColdTotalLifetimeThreshold + 1),
  96             AllocationType::Cold);
  97
  98   // Test NotCold
  99   // Long lived with more accesses per byte per sec than cold threshold is not cold.
 100   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold + 1, AllocCount,
 101                          ColdTotalLifetimeThreshold + 1),
 102             AllocationType::NotCold);
 103   // Short lived with more accesses per byte per sec than cold threshold is not cold.
 104   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold + 1, AllocCount,
 105                          ColdTotalLifetimeThreshold - 1),
 106             AllocationType::NotCold);
 107   // Short lived with less accesses per byte per sec than cold threshold is not cold.
 108   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold - 1, AllocCount,
 109                          ColdTotalLifetimeThreshold - 1),
 110             AllocationType::NotCold);
 111 }
 112
 113 // Test the hasSingleAllocType helper.
 114 TEST_F(MemoryProfileInfoTest, SingleAllocType) {
 115   uint8_t NotCold = (uint8_t)AllocationType::NotCold;
 116   uint8_t Cold = (uint8_t)AllocationType::Cold;
 117   uint8_t Hot = (uint8_t)AllocationType::Hot;
 118   EXPECT_TRUE(hasSingleAllocType(NotCold));
 119   EXPECT_TRUE(hasSingleAllocType(Cold));
 120   EXPECT_TRUE(hasSingleAllocType(Hot));
 121   EXPECT_FALSE(hasSingleAllocType(NotCold | Cold));
 122   EXPECT_FALSE(hasSingleAllocType(NotCold | Hot));
 123   EXPECT_FALSE(hasSingleAllocType(Cold | Hot));
 124   EXPECT_FALSE(hasSingleAllocType(NotCold | Cold | Hot));
 125 }
 126
 127 // Test buildCallstackMetadata helper.
 128 TEST_F(MemoryProfileInfoTest, BuildCallStackMD) {
 129   LLVMContext C;
 130   MDNode *CallStack = buildCallstackMetadata({1, 2, 3}, C);
 131   ASSERT_EQ(CallStack->getNumOperands(), 3u);
 132   unsigned ExpectedId = 1;
 133   for (auto &Op : CallStack->operands()) {
 134     auto *StackId = mdconst::dyn_extract<ConstantInt>(Op);
 135     EXPECT_EQ(StackId->getZExtValue(), ExpectedId++);
 136   }
 137 }
 138
 139 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 140 // call stack ids.
 141 // Check that allocations with a single allocation type along all call stacks
 142 // get an attribute instead of memprof metadata.
 143 TEST_F(MemoryProfileInfoTest, Attribute) {
 144   LLVMContext C;
 145   std::unique_ptr<Module> M = makeLLVMModule(C,
 146                                              R"IR(
 147 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 148 target triple = "x86_64-pc-linux-gnu"
 149 define i32* @test() {
 150 entry:
 151   %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 152   %0 = bitcast i8* %call1 to i32*
 153   %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 154   %1 = bitcast i8* %call2 to i32*
 155   %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 156   %2 = bitcast i8* %call3 to i32*
 157   ret i32* %1
 158 }
 159 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 160 )IR");
 161
 162   Function *Func = M->getFunction("test");
 163
 164   // First call has all cold contexts.
 165   CallStackTrie Trie1;
 166   Trie1.addCallStack(AllocationType::Cold, {1, 2});
 167   Trie1.addCallStack(AllocationType::Cold, {1, 3, 4});
 168   CallBase *Call1 = findCall(*Func, "call1");
 169   Trie1.buildAndAttachMIBMetadata(Call1);
 170
 171   EXPECT_FALSE(Call1->hasMetadata(LLVMContext::MD_memprof));
 172   EXPECT_TRUE(Call1->hasFnAttr("memprof"));
 173   EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");
 174
 175   // Second call has all non-cold contexts.
 176   CallStackTrie Trie2;
 177   Trie2.addCallStack(AllocationType::NotCold, {5, 6});
 178   Trie2.addCallStack(AllocationType::NotCold, {5, 7, 8});
 179   CallBase *Call2 = findCall(*Func, "call2");
 180   Trie2.buildAndAttachMIBMetadata(Call2);
 181
 182   EXPECT_FALSE(Call2->hasMetadata(LLVMContext::MD_memprof));
 183   EXPECT_TRUE(Call2->hasFnAttr("memprof"));
 184   EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");
 185
 186   // Third call has all hot contexts.
 187   CallStackTrie Trie3;
 188   Trie3.addCallStack(AllocationType::Hot, {9, 10});
 189   Trie3.addCallStack(AllocationType::Hot, {9, 11, 12});
 190   CallBase *Call3 = findCall(*Func, "call3");
 191   Trie3.buildAndAttachMIBMetadata(Call3);
 192
 193   EXPECT_FALSE(Call3->hasMetadata(LLVMContext::MD_memprof));
 194   EXPECT_TRUE(Call3->hasFnAttr("memprof"));
 195   EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");
 196 }
 197
 198 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 199 // call stack ids.
 200 // Test that an allocation call reached by both cold and non cold call stacks
 201 // gets memprof metadata representing the different allocation type contexts.
 202 TEST_F(MemoryProfileInfoTest, ColdAndNotColdMIB) {
 203   LLVMContext C;
 204   std::unique_ptr<Module> M = makeLLVMModule(C,
 205                                              R"IR(
 206 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 207 target triple = "x86_64-pc-linux-gnu"
 208 define i32* @test() {
 209 entry:
 210   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 211   %0 = bitcast i8* %call to i32*
 212   ret i32* %0
 213 }
 214 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 215 )IR");
 216
 217   Function *Func = M->getFunction("test");
 218
 219   CallStackTrie Trie;
 220   Trie.addCallStack(AllocationType::Cold, {1, 2});
 221   Trie.addCallStack(AllocationType::NotCold, {1, 3});
 222
 223   CallBase *Call = findCall(*Func, "call");
 224   Trie.buildAndAttachMIBMetadata(Call);
 225
 226   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 227   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 228   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 229   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 230   for (auto &MIBOp : MemProfMD->operands()) {
 231     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 232     MDNode *StackMD = getMIBStackNode(MIB);
 233     ASSERT_NE(StackMD, nullptr);
 234     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 235     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 236     ASSERT_EQ(StackId->getZExtValue(), 1u);
 237     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 238     if (StackId->getZExtValue() == 2u)
 239       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 240     else {
 241       ASSERT_EQ(StackId->getZExtValue(), 3u);
 242       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 243     }
 244   }
 245 }
 246
 247 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 248 // call stack ids.
 249 // Test that an allocation call reached by both cold and hot call stacks
 250 // gets memprof metadata representing the different allocation type contexts.
 251 TEST_F(MemoryProfileInfoTest, ColdAndHotMIB) {
 252   LLVMContext C;
 253   std::unique_ptr<Module> M = makeLLVMModule(C,
 254                                              R"IR(
 255 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 256 target triple = "x86_64-pc-linux-gnu"
 257 define i32* @test() {
 258 entry:
 259   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 260   %0 = bitcast i8* %call to i32*
 261   ret i32* %0
 262 }
 263 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 264 )IR");
 265
 266   Function *Func = M->getFunction("test");
 267
 268   CallStackTrie Trie;
 269   Trie.addCallStack(AllocationType::Cold, {1, 2});
 270   Trie.addCallStack(AllocationType::Hot, {1, 3});
 271
 272   CallBase *Call = findCall(*Func, "call");
 273   Trie.buildAndAttachMIBMetadata(Call);
 274
 275   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 276   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 277   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 278   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 279   for (auto &MIBOp : MemProfMD->operands()) {
 280     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 281     MDNode *StackMD = getMIBStackNode(MIB);
 282     ASSERT_NE(StackMD, nullptr);
 283     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 284     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 285     ASSERT_EQ(StackId->getZExtValue(), 1u);
 286     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 287     if (StackId->getZExtValue() == 2u)
 288       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 289     else {
 290       ASSERT_EQ(StackId->getZExtValue(), 3u);
 291       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 292     }
 293   }
 294 }
 295
 296 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 297 // call stack ids.
 298 // Test that an allocation call reached by both non cold and hot call stacks
 299 // gets memprof metadata representing the different allocation type contexts.
 300 TEST_F(MemoryProfileInfoTest, NotColdAndHotMIB) {
 301   LLVMContext C;
 302   std::unique_ptr<Module> M = makeLLVMModule(C,
 303                                              R"IR(
 304 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 305 target triple = "x86_64-pc-linux-gnu"
 306 define i32* @test() {
 307 entry:
 308   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 309   %0 = bitcast i8* %call to i32*
 310   ret i32* %0
 311 }
 312 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 313 )IR");
 314
 315   Function *Func = M->getFunction("test");
 316
 317   CallStackTrie Trie;
 318   Trie.addCallStack(AllocationType::NotCold, {1, 2});
 319   Trie.addCallStack(AllocationType::Hot, {1, 3});
 320
 321   CallBase *Call = findCall(*Func, "call");
 322   Trie.buildAndAttachMIBMetadata(Call);
 323
 324   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 325   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 326   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 327   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 328   for (auto &MIBOp : MemProfMD->operands()) {
 329     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 330     MDNode *StackMD = getMIBStackNode(MIB);
 331     ASSERT_NE(StackMD, nullptr);
 332     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 333     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 334     ASSERT_EQ(StackId->getZExtValue(), 1u);
 335     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 336     if (StackId->getZExtValue() == 2u)
 337       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 338     else {
 339       ASSERT_EQ(StackId->getZExtValue(), 3u);
 340       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 341     }
 342   }
 343 }
 344
 345 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 346 // call stack ids.
 347 // Test that an allocation call reached by both cold, non cold and hot call
 348 // stacks gets memprof metadata representing the different allocation type
 349 // contexts.
 350 TEST_F(MemoryProfileInfoTest, ColdAndNotColdAndHotMIB) {
 351   LLVMContext C;
 352   std::unique_ptr<Module> M = makeLLVMModule(C,
 353                                              R"IR(
 354 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 355 target triple = "x86_64-pc-linux-gnu"
 356 define i32* @test() {
 357 entry:
 358   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 359   %0 = bitcast i8* %call to i32*
 360   ret i32* %0
 361 }
 362 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 363 )IR");
 364
 365   Function *Func = M->getFunction("test");
 366
 367   CallStackTrie Trie;
 368   Trie.addCallStack(AllocationType::Cold, {1, 2});
 369   Trie.addCallStack(AllocationType::NotCold, {1, 3});
 370   Trie.addCallStack(AllocationType::Hot, {1, 4});
 371
 372   CallBase *Call = findCall(*Func, "call");
 373   Trie.buildAndAttachMIBMetadata(Call);
 374
 375   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 376   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 377   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 378   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 379   for (auto &MIBOp : MemProfMD->operands()) {
 380     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 381     MDNode *StackMD = getMIBStackNode(MIB);
 382     ASSERT_NE(StackMD, nullptr);
 383     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 384     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 385     ASSERT_EQ(StackId->getZExtValue(), 1u);
 386     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 387     if (StackId->getZExtValue() == 2u) {
 388       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 389     } else if (StackId->getZExtValue() == 3u) {
 390       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 391     } else {
 392       ASSERT_EQ(StackId->getZExtValue(), 4u);
 393       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 394     }
 395   }
 396 }
 397
 398 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 399 // call stack ids.
 400 // Test that an allocation call reached by multiple call stacks has memprof
 401 // metadata with the contexts trimmed to the minimum context required to
 402 // identify the allocation type.
 403 TEST_F(MemoryProfileInfoTest, TrimmedMIBContext) {
 404   LLVMContext C;
 405   std::unique_ptr<Module> M = makeLLVMModule(C,
 406                                              R"IR(
 407 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 408 target triple = "x86_64-pc-linux-gnu"
 409 define i32* @test() {
 410 entry:
 411   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 412   %0 = bitcast i8* %call to i32*
 413   ret i32* %0
 414 }
 415 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 416 )IR");
 417
 418   Function *Func = M->getFunction("test");
 419
 420   CallStackTrie Trie;
 421   // We should be able to trim the following two and combine into a single MIB
 422   // with the cold context {1, 2}.
 423   Trie.addCallStack(AllocationType::Cold, {1, 2, 3});
 424   Trie.addCallStack(AllocationType::Cold, {1, 2, 4});
 425   // We should be able to trim the following two and combine into a single MIB
 426   // with the non-cold context {1, 5}.
 427   Trie.addCallStack(AllocationType::NotCold, {1, 5, 6});
 428   Trie.addCallStack(AllocationType::NotCold, {1, 5, 7});
 429   // We should be able to trim the following two and combine into a single MIB
 430   // with the hot context {1, 8}.
 431   Trie.addCallStack(AllocationType::Hot, {1, 8, 9});
 432   Trie.addCallStack(AllocationType::Hot, {1, 8, 10});
 433
 434   CallBase *Call = findCall(*Func, "call");
 435   Trie.buildAndAttachMIBMetadata(Call);
 436
 437   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 438   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 439   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 440   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 441   for (auto &MIBOp : MemProfMD->operands()) {
 442     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 443     MDNode *StackMD = getMIBStackNode(MIB);
 444     ASSERT_NE(StackMD, nullptr);
 445     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 446     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 447     EXPECT_EQ(StackId->getZExtValue(), 1u);
 448     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 449     if (StackId->getZExtValue() == 2u)
 450       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 451     else if (StackId->getZExtValue() == 5u)
 452       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 453     else {
 454       ASSERT_EQ(StackId->getZExtValue(), 8u);
 455       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 456     }
 457   }
 458 }
 459
 460 // Test CallStackTrie::addCallStack interface taking memprof MIB metadata.
 461 // Check that allocations annotated with memprof metadata with a single
 462 // allocation type get simplified to an attribute.
 463 TEST_F(MemoryProfileInfoTest, SimplifyMIBToAttribute) {
 464   LLVMContext C;
 465   std::unique_ptr<Module> M = makeLLVMModule(C,
 466                                              R"IR(
 467 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 468 target triple = "x86_64-pc-linux-gnu"
 469 define i32* @test() {
 470 entry:
 471   %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0
 472   %0 = bitcast i8* %call1 to i32*
 473   %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !3
 474   %1 = bitcast i8* %call2 to i32*
 475   %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !6
 476   %2 = bitcast i8* %call3 to i32*
 477   ret i32* %1
 478 }
 479 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 480 !0 = !{!1}
 481 !1 = !{!2, !"cold"}
 482 !2 = !{i64 1, i64 2, i64 3}
 483 !3 = !{!4}
 484 !4 = !{!5, !"notcold"}
 485 !5 = !{i64 4, i64 5, i64 6, i64 7}
 486 !6 = !{!7}
 487 !7 = !{!8, !"hot"}
 488 !8 = !{i64 8, i64 9, i64 10}
 489 )IR");
 490
 491   Function *Func = M->getFunction("test");
 492
 493   // First call has all cold contexts.
 494   CallStackTrie Trie1;
 495   CallBase *Call1 = findCall(*Func, "call1");
 496   MDNode *MemProfMD1 = Call1->getMetadata(LLVMContext::MD_memprof);
 497   ASSERT_EQ(MemProfMD1->getNumOperands(), 1u);
 498   MDNode *MIB1 = dyn_cast<MDNode>(MemProfMD1->getOperand(0));
 499   Trie1.addCallStack(MIB1);
 500   Trie1.buildAndAttachMIBMetadata(Call1);
 501
 502   EXPECT_TRUE(Call1->hasFnAttr("memprof"));
 503   EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");
 504
 505   // Second call has all non-cold contexts.
 506   CallStackTrie Trie2;
 507   CallBase *Call2 = findCall(*Func, "call2");
 508   MDNode *MemProfMD2 = Call2->getMetadata(LLVMContext::MD_memprof);
 509   ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);
 510   MDNode *MIB2 = dyn_cast<MDNode>(MemProfMD2->getOperand(0));
 511   Trie2.addCallStack(MIB2);
 512   Trie2.buildAndAttachMIBMetadata(Call2);
 513
 514   EXPECT_TRUE(Call2->hasFnAttr("memprof"));
 515   EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");
 516
 517   // Third call has all hot contexts.
 518   CallStackTrie Trie3;
 519   CallBase *Call3 = findCall(*Func, "call3");
 520   MDNode *MemProfMD3 = Call3->getMetadata(LLVMContext::MD_memprof);
 521   ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);
 522   MDNode *MIB3 = dyn_cast<MDNode>(MemProfMD3->getOperand(0));
 523   Trie3.addCallStack(MIB3);
 524   Trie3.buildAndAttachMIBMetadata(Call3);
 525
 526   EXPECT_TRUE(Call3->hasFnAttr("memprof"));
 527   EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");
 528 }
 529
 530 // Test CallStackTrie::addCallStack interface taking memprof MIB metadata.
 531 // Test that allocations annotated with memprof metadata with multiple call
 532 // stacks gets new memprof metadata with the contexts trimmed to the minimum
 533 // context required to identify the allocation type.
 534 TEST_F(MemoryProfileInfoTest, ReTrimMIBContext) {
 535   LLVMContext C;
 536   std::unique_ptr<Module> M = makeLLVMModule(C,
 537                                              R"IR(
 538 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 539 target triple = "x86_64-pc-linux-gnu"
 540 define i32* @test() {
 541 entry:
 542   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0
 543   %0 = bitcast i8* %call to i32*
 544   ret i32* %0
 545 }
 546 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 547 !0 = !{!1, !3, !5, !7, !9, !11}
 548 !1 = !{!2, !"cold"}
 549 !2 = !{i64 1, i64 2, i64 3}
 550 !3 = !{!4, !"cold"}
 551 !4 = !{i64 1, i64 2, i64 4}
 552 !5 = !{!6, !"notcold"}
 553 !6 = !{i64 1, i64 5, i64 6}
 554 !7 = !{!8, !"notcold"}
 555 !8 = !{i64 1, i64 5, i64 7}
 556 !9 = !{!10, !"hot"}
 557 !10 = !{i64 1, i64 8, i64 9}
 558 !11 = !{!12, !"hot"}
 559 !12 = !{i64 1, i64 8, i64 10}
 560 )IR");
 561
 562   Function *Func = M->getFunction("test");
 563
 564   CallStackTrie Trie;
 565   ASSERT_TRUE(Trie.empty());
 566   CallBase *Call = findCall(*Func, "call");
 567   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 568   for (auto &MIBOp : MemProfMD->operands()) {
 569     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 570     Trie.addCallStack(MIB);
 571   }
 572   ASSERT_FALSE(Trie.empty());
 573   Trie.buildAndAttachMIBMetadata(Call);
 574
 575   // We should be able to trim the first two and combine into a single MIB
 576   // with the cold context {1, 2}.
 577   // We should be able to trim the second two and combine into a single MIB
 578   // with the non-cold context {1, 5}.
 579
 580   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 581   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 582   MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 583   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 584   for (auto &MIBOp : MemProfMD->operands()) {
 585     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 586     MDNode *StackMD = getMIBStackNode(MIB);
 587     ASSERT_NE(StackMD, nullptr);
 588     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 589     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 590     EXPECT_EQ(StackId->getZExtValue(), 1u);
 591     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 592     if (StackId->getZExtValue() == 2u)
 593       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 594     else if (StackId->getZExtValue() == 5u)
 595       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 596     else {
 597       ASSERT_EQ(StackId->getZExtValue(), 8u);
 598       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 599     }
 600   }
 601 }
 602
 603 TEST_F(MemoryProfileInfoTest, CallStackTestIR) {
 604   LLVMContext C;
 605   std::unique_ptr<Module> M = makeLLVMModule(C,
 606                                              R"IR(
 607 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 608 target triple = "x86_64-pc-linux-gnu"
 609 define ptr @test() {
 610 entry:
 611   %call = call noalias noundef nonnull dereferenceable(10) ptr @_Znam(i64 noundef 10), !memprof !1, !callsite !6
 612   ret ptr %call
 613 }
 614 declare noundef nonnull ptr @_Znam(i64 noundef)
 615 !1 = !{!2, !4, !7}
 616 !2 = !{!3, !"notcold"}
 617 !3 = !{i64 1, i64 2, i64 3, i64 4}
 618 !4 = !{!5, !"cold"}
 619 !5 = !{i64 1, i64 2, i64 3, i64 5}
 620 !6 = !{i64 1}
 621 !7 = !{!8, !"hot"}
 622 !8 = !{i64 1, i64 2, i64 3, i64 6}
 623 )IR");
 624
 625   Function *Func = M->getFunction("test");
 626   CallBase *Call = findCall(*Func, "call");
 627
 628   CallStack<MDNode, MDNode::op_iterator> InstCallsite(
 629       Call->getMetadata(LLVMContext::MD_callsite));
 630
 631   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 632   unsigned Idx = 0;
 633   for (auto &MIBOp : MemProfMD->operands()) {
 634     auto *MIBMD = cast<const MDNode>(MIBOp);
 635     MDNode *StackNode = getMIBStackNode(MIBMD);
 636     CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
 637     EXPECT_EQ(StackContext.back(), 4 + Idx);
 638     std::vector<uint64_t> StackIds;
 639     for (auto ContextIter = StackContext.beginAfterSharedPrefix(InstCallsite);
 640          ContextIter != StackContext.end(); ++ContextIter)
 641       StackIds.push_back(*ContextIter);
 642     if (Idx == 0) {
 643       std::vector<uint64_t> Expected = {2, 3, 4};
 644       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 645     } else if (Idx == 1) {
 646       std::vector<uint64_t> Expected = {2, 3, 5};
 647       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 648     } else {
 649       std::vector<uint64_t> Expected = {2, 3, 6};
 650       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 651     }
 652     Idx++;
 653   }
 654 }
 655
 656 TEST_F(MemoryProfileInfoTest, CallStackTestSummary) {
 657   std::unique_ptr<ModuleSummaryIndex> Index = makeLLVMIndex(R"Summary(
 658 ^0 = module: (path: "test.o", hash: (0, 0, 0, 0, 0))
 659 ^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (1, 2, 3, 4)), (type: cold, stackIds: (1, 2, 3, 5)), (type: hot, stackIds: (1, 2, 3, 6))))))))
 660 ^2 = gv: (guid: 25, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1)), callsites: ((callee: ^1, clones: (0), stackIds: (3, 4)), (callee: ^1, clones: (0), stackIds: (3, 5)), (callee: ^1, clones: (0), stackIds: (3, 6))))))
 661 )Summary");
 662
 663   ASSERT_NE(Index, nullptr);
 664   auto *CallsiteSummary =
 665       cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/25));
 666   unsigned Idx = 0;
 667   for (auto &CI : CallsiteSummary->callsites()) {
 668     CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator> InstCallsite(
 669         &CI);
 670     std::vector<uint64_t> StackIds;
 671     for (auto StackIdIndex : InstCallsite)
 672       StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));
 673     if (Idx == 0) {
 674       std::vector<uint64_t> Expected = {3, 4};
 675       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 676     } else if (Idx == 1) {
 677       std::vector<uint64_t> Expected = {3, 5};
 678       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 679     } else {
 680       std::vector<uint64_t> Expected = {3, 6};
 681       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 682     }
 683     Idx++;
 684   }
 685
 686   auto *AllocSummary =
 687       cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/23));
 688   for (auto &AI : AllocSummary->allocs()) {
 689     unsigned Idx = 0;
 690     for (auto &MIB : AI.MIBs) {
 691       CallStack<MIBInfo, SmallVector<unsigned>::const_iterator> StackContext(
 692           &MIB);
 693       EXPECT_EQ(Index->getStackIdAtIndex(StackContext.back()), 4 + Idx);
 694       std::vector<uint64_t> StackIds;
 695       for (auto StackIdIndex : StackContext)
 696         StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));
 697       if (Idx == 0) {
 698         std::vector<uint64_t> Expected = {1, 2, 3, 4};
 699         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 700       } else if (Idx == 1) {
 701         std::vector<uint64_t> Expected = {1, 2, 3, 5};
 702         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 703       } else {
 704         std::vector<uint64_t> Expected = {1, 2, 3, 6};
 705         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 706       }
 707       Idx++;
 708     }
 709   }
 710 }
 711 } // end anonymous namespace