llvm/unittests/Analysis/MemoryProfileInfoTest.cpp

   1 //===- MemoryProfileInfoTest.cpp - Memory Profile Info Unit Tests-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/MemoryProfileInfo.h"
  10 #include "llvm/AsmParser/Parser.h"
  11 #include "llvm/IR/Constants.h"
  12 #include "llvm/IR/Instructions.h"
  13 #include "llvm/IR/LLVMContext.h"
  14 #include "llvm/IR/Module.h"
  15 #include "llvm/IR/ModuleSummaryIndex.h"
  16 #include "llvm/Support/CommandLine.h"
  17 #include "llvm/Support/SourceMgr.h"
  18 #include "gtest/gtest.h"
  19 #include <cstring>
  20 #include <sys/types.h>
  21
  22 using namespace llvm;
  23 using namespace llvm::memprof;
  24
  25 extern cl::opt<float> MemProfLifetimeAccessDensityColdThreshold;
  26 extern cl::opt<unsigned> MemProfAveLifetimeColdThreshold;
  27 extern cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold;
  28
  29 namespace {
  30
  31 class MemoryProfileInfoTest : public testing::Test {
  32 protected:
  33   std::unique_ptr<Module> makeLLVMModule(LLVMContext &C, const char *IR) {
  34     SMDiagnostic Err;
  35     std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
  36     if (!Mod)
  37       Err.print("MemoryProfileInfoTest", errs());
  38     return Mod;
  39   }
  40
  41   std::unique_ptr<ModuleSummaryIndex> makeLLVMIndex(const char *Summary) {
  42     SMDiagnostic Err;
  43     std::unique_ptr<ModuleSummaryIndex> Index =
  44         parseSummaryIndexAssemblyString(Summary, Err);
  45     if (!Index)
  46       Err.print("MemoryProfileInfoTest", errs());
  47     return Index;
  48   }
  49
  50   // This looks for a call that has the given value name, which
  51   // is the name of the value being assigned the call return value.
  52   CallBase *findCall(Function &F, const char *Name = nullptr) {
  53     for (auto &BB : F)
  54       for (auto &I : BB)
  55         if (auto *CB = dyn_cast<CallBase>(&I))
  56           if (!Name || CB->getName() == Name)
  57             return CB;
  58     return nullptr;
  59   }
  60 };
  61
  62 // Test getAllocType helper.
  63 // Basic checks on the allocation type for values just above and below
  64 // the thresholds.
  65 TEST_F(MemoryProfileInfoTest, GetAllocType) {
  66   const uint64_t AllocCount = 2;
  67   // To be cold we require that
  68   // ((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
  69   //    MemProfLifetimeAccessDensityColdThreshold
  70   // so compute the ColdTotalLifetimeAccessDensityThreshold at the threshold.
  71   const uint64_t ColdTotalLifetimeAccessDensityThreshold =
  72       (uint64_t)(MemProfLifetimeAccessDensityColdThreshold * AllocCount * 100);
  73   // To be cold we require that
  74   // ((float)TotalLifetime) / AllocCount >=
  75   //    MemProfAveLifetimeColdThreshold * 1000
  76   // so compute the TotalLifetime right at the threshold.
  77   const uint64_t ColdTotalLifetimeThreshold =
  78       MemProfAveLifetimeColdThreshold * AllocCount * 1000;
  79   // To be hot we require that
  80   // ((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
  81   //    MemProfMinAveLifetimeAccessDensityHotThreshold
  82   // so compute the HotTotalLifetimeAccessDensityThreshold  at the threshold.
  83   const uint64_t HotTotalLifetimeAccessDensityThreshold =
  84       (uint64_t)(MemProfMinAveLifetimeAccessDensityHotThreshold * AllocCount * 100);
  85
  86
  87   // Test Hot
  88   // More accesses per byte per sec than hot threshold is hot.
  89   EXPECT_EQ(getAllocType(HotTotalLifetimeAccessDensityThreshold + 1, AllocCount,
  90                          ColdTotalLifetimeThreshold + 1),
  91             AllocationType::Hot);
  92
  93   // Test Cold
  94   // Long lived with less accesses per byte per sec than cold threshold is cold.
  95   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold - 1, AllocCount,
  96                          ColdTotalLifetimeThreshold + 1),
  97             AllocationType::Cold);
  98
  99   // Test NotCold
 100   // Long lived with more accesses per byte per sec than cold threshold is not cold.
 101   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold + 1, AllocCount,
 102                          ColdTotalLifetimeThreshold + 1),
 103             AllocationType::NotCold);
 104   // Short lived with more accesses per byte per sec than cold threshold is not cold.
 105   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold + 1, AllocCount,
 106                          ColdTotalLifetimeThreshold - 1),
 107             AllocationType::NotCold);
 108   // Short lived with less accesses per byte per sec than cold threshold is not cold.
 109   EXPECT_EQ(getAllocType(ColdTotalLifetimeAccessDensityThreshold - 1, AllocCount,
 110                          ColdTotalLifetimeThreshold - 1),
 111             AllocationType::NotCold);
 112 }
 113
 114 // Test the hasSingleAllocType helper.
 115 TEST_F(MemoryProfileInfoTest, SingleAllocType) {
 116   uint8_t NotCold = (uint8_t)AllocationType::NotCold;
 117   uint8_t Cold = (uint8_t)AllocationType::Cold;
 118   uint8_t Hot = (uint8_t)AllocationType::Hot;
 119   EXPECT_TRUE(hasSingleAllocType(NotCold));
 120   EXPECT_TRUE(hasSingleAllocType(Cold));
 121   EXPECT_TRUE(hasSingleAllocType(Hot));
 122   EXPECT_FALSE(hasSingleAllocType(NotCold | Cold));
 123   EXPECT_FALSE(hasSingleAllocType(NotCold | Hot));
 124   EXPECT_FALSE(hasSingleAllocType(Cold | Hot));
 125   EXPECT_FALSE(hasSingleAllocType(NotCold | Cold | Hot));
 126 }
 127
 128 // Test buildCallstackMetadata helper.
 129 TEST_F(MemoryProfileInfoTest, BuildCallStackMD) {
 130   LLVMContext C;
 131   MDNode *CallStack = buildCallstackMetadata({1, 2, 3}, C);
 132   ASSERT_EQ(CallStack->getNumOperands(), 3u);
 133   unsigned ExpectedId = 1;
 134   for (auto &Op : CallStack->operands()) {
 135     auto *StackId = mdconst::dyn_extract<ConstantInt>(Op);
 136     EXPECT_EQ(StackId->getZExtValue(), ExpectedId++);
 137   }
 138 }
 139
 140 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 141 // call stack ids.
 142 // Check that allocations with a single allocation type along all call stacks
 143 // get an attribute instead of memprof metadata.
 144 TEST_F(MemoryProfileInfoTest, Attribute) {
 145   LLVMContext C;
 146   std::unique_ptr<Module> M = makeLLVMModule(C,
 147                                              R"IR(
 148 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 149 target triple = "x86_64-pc-linux-gnu"
 150 define i32* @test() {
 151 entry:
 152   %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 153   %0 = bitcast i8* %call1 to i32*
 154   %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 155   %1 = bitcast i8* %call2 to i32*
 156   %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 157   %2 = bitcast i8* %call3 to i32*
 158   ret i32* %1
 159 }
 160 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 161 )IR");
 162
 163   Function *Func = M->getFunction("test");
 164
 165   // First call has all cold contexts.
 166   CallStackTrie Trie1;
 167   Trie1.addCallStack(AllocationType::Cold, {1, 2});
 168   Trie1.addCallStack(AllocationType::Cold, {1, 3, 4});
 169   CallBase *Call1 = findCall(*Func, "call1");
 170   Trie1.buildAndAttachMIBMetadata(Call1);
 171
 172   EXPECT_FALSE(Call1->hasMetadata(LLVMContext::MD_memprof));
 173   EXPECT_TRUE(Call1->hasFnAttr("memprof"));
 174   EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");
 175
 176   // Second call has all non-cold contexts.
 177   CallStackTrie Trie2;
 178   Trie2.addCallStack(AllocationType::NotCold, {5, 6});
 179   Trie2.addCallStack(AllocationType::NotCold, {5, 7, 8});
 180   CallBase *Call2 = findCall(*Func, "call2");
 181   Trie2.buildAndAttachMIBMetadata(Call2);
 182
 183   EXPECT_FALSE(Call2->hasMetadata(LLVMContext::MD_memprof));
 184   EXPECT_TRUE(Call2->hasFnAttr("memprof"));
 185   EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");
 186
 187   // Third call has all hot contexts.
 188   CallStackTrie Trie3;
 189   Trie3.addCallStack(AllocationType::Hot, {9, 10});
 190   Trie3.addCallStack(AllocationType::Hot, {9, 11, 12});
 191   CallBase *Call3 = findCall(*Func, "call3");
 192   Trie3.buildAndAttachMIBMetadata(Call3);
 193
 194   EXPECT_FALSE(Call3->hasMetadata(LLVMContext::MD_memprof));
 195   EXPECT_TRUE(Call3->hasFnAttr("memprof"));
 196   EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");
 197 }
 198
 199 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 200 // call stack ids.
 201 // Test that an allocation call reached by both cold and non cold call stacks
 202 // gets memprof metadata representing the different allocation type contexts.
 203 TEST_F(MemoryProfileInfoTest, ColdAndNotColdMIB) {
 204   LLVMContext C;
 205   std::unique_ptr<Module> M = makeLLVMModule(C,
 206                                              R"IR(
 207 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 208 target triple = "x86_64-pc-linux-gnu"
 209 define i32* @test() {
 210 entry:
 211   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 212   %0 = bitcast i8* %call to i32*
 213   ret i32* %0
 214 }
 215 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 216 )IR");
 217
 218   Function *Func = M->getFunction("test");
 219
 220   CallStackTrie Trie;
 221   Trie.addCallStack(AllocationType::Cold, {1, 2});
 222   Trie.addCallStack(AllocationType::NotCold, {1, 3});
 223
 224   CallBase *Call = findCall(*Func, "call");
 225   Trie.buildAndAttachMIBMetadata(Call);
 226
 227   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 228   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 229   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 230   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 231   for (auto &MIBOp : MemProfMD->operands()) {
 232     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 233     MDNode *StackMD = getMIBStackNode(MIB);
 234     ASSERT_NE(StackMD, nullptr);
 235     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 236     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 237     ASSERT_EQ(StackId->getZExtValue(), 1u);
 238     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 239     if (StackId->getZExtValue() == 2u)
 240       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 241     else {
 242       ASSERT_EQ(StackId->getZExtValue(), 3u);
 243       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 244     }
 245   }
 246 }
 247
 248 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 249 // call stack ids.
 250 // Test that an allocation call reached by both cold and hot call stacks
 251 // gets memprof metadata representing the different allocation type contexts.
 252 TEST_F(MemoryProfileInfoTest, ColdAndHotMIB) {
 253   LLVMContext C;
 254   std::unique_ptr<Module> M = makeLLVMModule(C,
 255                                              R"IR(
 256 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 257 target triple = "x86_64-pc-linux-gnu"
 258 define i32* @test() {
 259 entry:
 260   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 261   %0 = bitcast i8* %call to i32*
 262   ret i32* %0
 263 }
 264 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 265 )IR");
 266
 267   Function *Func = M->getFunction("test");
 268
 269   CallStackTrie Trie;
 270   Trie.addCallStack(AllocationType::Cold, {1, 2});
 271   Trie.addCallStack(AllocationType::Hot, {1, 3});
 272
 273   CallBase *Call = findCall(*Func, "call");
 274   Trie.buildAndAttachMIBMetadata(Call);
 275
 276   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 277   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 278   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 279   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 280   for (auto &MIBOp : MemProfMD->operands()) {
 281     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 282     MDNode *StackMD = getMIBStackNode(MIB);
 283     ASSERT_NE(StackMD, nullptr);
 284     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 285     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 286     ASSERT_EQ(StackId->getZExtValue(), 1u);
 287     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 288     if (StackId->getZExtValue() == 2u)
 289       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 290     else {
 291       ASSERT_EQ(StackId->getZExtValue(), 3u);
 292       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 293     }
 294   }
 295 }
 296
 297 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 298 // call stack ids.
 299 // Test that an allocation call reached by both non cold and hot call stacks
 300 // gets memprof metadata representing the different allocation type contexts.
 301 TEST_F(MemoryProfileInfoTest, NotColdAndHotMIB) {
 302   LLVMContext C;
 303   std::unique_ptr<Module> M = makeLLVMModule(C,
 304                                              R"IR(
 305 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 306 target triple = "x86_64-pc-linux-gnu"
 307 define i32* @test() {
 308 entry:
 309   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 310   %0 = bitcast i8* %call to i32*
 311   ret i32* %0
 312 }
 313 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 314 )IR");
 315
 316   Function *Func = M->getFunction("test");
 317
 318   CallStackTrie Trie;
 319   Trie.addCallStack(AllocationType::NotCold, {1, 2});
 320   Trie.addCallStack(AllocationType::Hot, {1, 3});
 321
 322   CallBase *Call = findCall(*Func, "call");
 323   Trie.buildAndAttachMIBMetadata(Call);
 324
 325   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 326   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 327   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 328   ASSERT_EQ(MemProfMD->getNumOperands(), 2u);
 329   for (auto &MIBOp : MemProfMD->operands()) {
 330     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 331     MDNode *StackMD = getMIBStackNode(MIB);
 332     ASSERT_NE(StackMD, nullptr);
 333     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 334     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 335     ASSERT_EQ(StackId->getZExtValue(), 1u);
 336     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 337     if (StackId->getZExtValue() == 2u)
 338       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 339     else {
 340       ASSERT_EQ(StackId->getZExtValue(), 3u);
 341       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 342     }
 343   }
 344 }
 345
 346 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 347 // call stack ids.
 348 // Test that an allocation call reached by both cold, non cold and hot call
 349 // stacks gets memprof metadata representing the different allocation type
 350 // contexts.
 351 TEST_F(MemoryProfileInfoTest, ColdAndNotColdAndHotMIB) {
 352   LLVMContext C;
 353   std::unique_ptr<Module> M = makeLLVMModule(C,
 354                                              R"IR(
 355 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 356 target triple = "x86_64-pc-linux-gnu"
 357 define i32* @test() {
 358 entry:
 359   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 360   %0 = bitcast i8* %call to i32*
 361   ret i32* %0
 362 }
 363 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 364 )IR");
 365
 366   Function *Func = M->getFunction("test");
 367
 368   CallStackTrie Trie;
 369   Trie.addCallStack(AllocationType::Cold, {1, 2});
 370   Trie.addCallStack(AllocationType::NotCold, {1, 3});
 371   Trie.addCallStack(AllocationType::Hot, {1, 4});
 372
 373   CallBase *Call = findCall(*Func, "call");
 374   Trie.buildAndAttachMIBMetadata(Call);
 375
 376   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 377   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 378   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 379   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 380   for (auto &MIBOp : MemProfMD->operands()) {
 381     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 382     MDNode *StackMD = getMIBStackNode(MIB);
 383     ASSERT_NE(StackMD, nullptr);
 384     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 385     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 386     ASSERT_EQ(StackId->getZExtValue(), 1u);
 387     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 388     if (StackId->getZExtValue() == 2u) {
 389       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 390     } else if (StackId->getZExtValue() == 3u) {
 391       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 392     } else {
 393       ASSERT_EQ(StackId->getZExtValue(), 4u);
 394       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 395     }
 396   }
 397 }
 398
 399 // Test CallStackTrie::addCallStack interface taking allocation type and list of
 400 // call stack ids.
 401 // Test that an allocation call reached by multiple call stacks has memprof
 402 // metadata with the contexts trimmed to the minimum context required to
 403 // identify the allocation type.
 404 TEST_F(MemoryProfileInfoTest, TrimmedMIBContext) {
 405   LLVMContext C;
 406   std::unique_ptr<Module> M = makeLLVMModule(C,
 407                                              R"IR(
 408 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 409 target triple = "x86_64-pc-linux-gnu"
 410 define i32* @test() {
 411 entry:
 412   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)
 413   %0 = bitcast i8* %call to i32*
 414   ret i32* %0
 415 }
 416 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 417 )IR");
 418
 419   Function *Func = M->getFunction("test");
 420
 421   CallStackTrie Trie;
 422   // We should be able to trim the following two and combine into a single MIB
 423   // with the cold context {1, 2}.
 424   Trie.addCallStack(AllocationType::Cold, {1, 2, 3});
 425   Trie.addCallStack(AllocationType::Cold, {1, 2, 4});
 426   // We should be able to trim the following two and combine into a single MIB
 427   // with the non-cold context {1, 5}.
 428   Trie.addCallStack(AllocationType::NotCold, {1, 5, 6});
 429   Trie.addCallStack(AllocationType::NotCold, {1, 5, 7});
 430   // We should be able to trim the following two and combine into a single MIB
 431   // with the hot context {1, 8}.
 432   Trie.addCallStack(AllocationType::Hot, {1, 8, 9});
 433   Trie.addCallStack(AllocationType::Hot, {1, 8, 10});
 434
 435   CallBase *Call = findCall(*Func, "call");
 436   Trie.buildAndAttachMIBMetadata(Call);
 437
 438   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 439   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 440   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 441   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 442   for (auto &MIBOp : MemProfMD->operands()) {
 443     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 444     MDNode *StackMD = getMIBStackNode(MIB);
 445     ASSERT_NE(StackMD, nullptr);
 446     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 447     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 448     EXPECT_EQ(StackId->getZExtValue(), 1u);
 449     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 450     if (StackId->getZExtValue() == 2u)
 451       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 452     else if (StackId->getZExtValue() == 5u)
 453       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 454     else {
 455       ASSERT_EQ(StackId->getZExtValue(), 8u);
 456       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 457     }
 458   }
 459 }
 460
 461 // Test CallStackTrie::addCallStack interface taking memprof MIB metadata.
 462 // Check that allocations annotated with memprof metadata with a single
 463 // allocation type get simplified to an attribute.
 464 TEST_F(MemoryProfileInfoTest, SimplifyMIBToAttribute) {
 465   LLVMContext C;
 466   std::unique_ptr<Module> M = makeLLVMModule(C,
 467                                              R"IR(
 468 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 469 target triple = "x86_64-pc-linux-gnu"
 470 define i32* @test() {
 471 entry:
 472   %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0
 473   %0 = bitcast i8* %call1 to i32*
 474   %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !3
 475   %1 = bitcast i8* %call2 to i32*
 476   %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !6
 477   %2 = bitcast i8* %call3 to i32*
 478   ret i32* %1
 479 }
 480 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 481 !0 = !{!1}
 482 !1 = !{!2, !"cold"}
 483 !2 = !{i64 1, i64 2, i64 3}
 484 !3 = !{!4}
 485 !4 = !{!5, !"notcold"}
 486 !5 = !{i64 4, i64 5, i64 6, i64 7}
 487 !6 = !{!7}
 488 !7 = !{!8, !"hot"}
 489 !8 = !{i64 8, i64 9, i64 10}
 490 )IR");
 491
 492   Function *Func = M->getFunction("test");
 493
 494   // First call has all cold contexts.
 495   CallStackTrie Trie1;
 496   CallBase *Call1 = findCall(*Func, "call1");
 497   MDNode *MemProfMD1 = Call1->getMetadata(LLVMContext::MD_memprof);
 498   ASSERT_EQ(MemProfMD1->getNumOperands(), 1u);
 499   MDNode *MIB1 = dyn_cast<MDNode>(MemProfMD1->getOperand(0));
 500   Trie1.addCallStack(MIB1);
 501   Trie1.buildAndAttachMIBMetadata(Call1);
 502
 503   EXPECT_TRUE(Call1->hasFnAttr("memprof"));
 504   EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");
 505
 506   // Second call has all non-cold contexts.
 507   CallStackTrie Trie2;
 508   CallBase *Call2 = findCall(*Func, "call2");
 509   MDNode *MemProfMD2 = Call2->getMetadata(LLVMContext::MD_memprof);
 510   ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);
 511   MDNode *MIB2 = dyn_cast<MDNode>(MemProfMD2->getOperand(0));
 512   Trie2.addCallStack(MIB2);
 513   Trie2.buildAndAttachMIBMetadata(Call2);
 514
 515   EXPECT_TRUE(Call2->hasFnAttr("memprof"));
 516   EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");
 517
 518   // Third call has all hot contexts.
 519   CallStackTrie Trie3;
 520   CallBase *Call3 = findCall(*Func, "call3");
 521   MDNode *MemProfMD3 = Call3->getMetadata(LLVMContext::MD_memprof);
 522   ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);
 523   MDNode *MIB3 = dyn_cast<MDNode>(MemProfMD3->getOperand(0));
 524   Trie3.addCallStack(MIB3);
 525   Trie3.buildAndAttachMIBMetadata(Call3);
 526
 527   EXPECT_TRUE(Call3->hasFnAttr("memprof"));
 528   EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");
 529 }
 530
 531 // Test CallStackTrie::addCallStack interface taking memprof MIB metadata.
 532 // Test that allocations annotated with memprof metadata with multiple call
 533 // stacks gets new memprof metadata with the contexts trimmed to the minimum
 534 // context required to identify the allocation type.
 535 TEST_F(MemoryProfileInfoTest, ReTrimMIBContext) {
 536   LLVMContext C;
 537   std::unique_ptr<Module> M = makeLLVMModule(C,
 538                                              R"IR(
 539 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 540 target triple = "x86_64-pc-linux-gnu"
 541 define i32* @test() {
 542 entry:
 543   %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0
 544   %0 = bitcast i8* %call to i32*
 545   ret i32* %0
 546 }
 547 declare dso_local noalias noundef i8* @malloc(i64 noundef)
 548 !0 = !{!1, !3, !5, !7, !9, !11}
 549 !1 = !{!2, !"cold"}
 550 !2 = !{i64 1, i64 2, i64 3}
 551 !3 = !{!4, !"cold"}
 552 !4 = !{i64 1, i64 2, i64 4}
 553 !5 = !{!6, !"notcold"}
 554 !6 = !{i64 1, i64 5, i64 6}
 555 !7 = !{!8, !"notcold"}
 556 !8 = !{i64 1, i64 5, i64 7}
 557 !9 = !{!10, !"hot"}
 558 !10 = !{i64 1, i64 8, i64 9}
 559 !11 = !{!12, !"hot"}
 560 !12 = !{i64 1, i64 8, i64 10}
 561 )IR");
 562
 563   Function *Func = M->getFunction("test");
 564
 565   CallStackTrie Trie;
 566   ASSERT_TRUE(Trie.empty());
 567   CallBase *Call = findCall(*Func, "call");
 568   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 569   for (auto &MIBOp : MemProfMD->operands()) {
 570     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 571     Trie.addCallStack(MIB);
 572   }
 573   ASSERT_FALSE(Trie.empty());
 574   Trie.buildAndAttachMIBMetadata(Call);
 575
 576   // We should be able to trim the first two and combine into a single MIB
 577   // with the cold context {1, 2}.
 578   // We should be able to trim the second two and combine into a single MIB
 579   // with the non-cold context {1, 5}.
 580
 581   EXPECT_FALSE(Call->hasFnAttr("memprof"));
 582   EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));
 583   MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 584   ASSERT_EQ(MemProfMD->getNumOperands(), 3u);
 585   for (auto &MIBOp : MemProfMD->operands()) {
 586     MDNode *MIB = dyn_cast<MDNode>(MIBOp);
 587     MDNode *StackMD = getMIBStackNode(MIB);
 588     ASSERT_NE(StackMD, nullptr);
 589     ASSERT_EQ(StackMD->getNumOperands(), 2u);
 590     auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));
 591     EXPECT_EQ(StackId->getZExtValue(), 1u);
 592     StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));
 593     if (StackId->getZExtValue() == 2u)
 594       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);
 595     else if (StackId->getZExtValue() == 5u)
 596       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);
 597     else {
 598       ASSERT_EQ(StackId->getZExtValue(), 8u);
 599       EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Hot);
 600     }
 601   }
 602 }
 603
 604 TEST_F(MemoryProfileInfoTest, CallStackTestIR) {
 605   LLVMContext C;
 606   std::unique_ptr<Module> M = makeLLVMModule(C,
 607                                              R"IR(
 608 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 609 target triple = "x86_64-pc-linux-gnu"
 610 define ptr @test() {
 611 entry:
 612   %call = call noalias noundef nonnull dereferenceable(10) ptr @_Znam(i64 noundef 10), !memprof !1, !callsite !6
 613   ret ptr %call
 614 }
 615 declare noundef nonnull ptr @_Znam(i64 noundef)
 616 !1 = !{!2, !4, !7}
 617 !2 = !{!3, !"notcold"}
 618 !3 = !{i64 1, i64 2, i64 3, i64 4}
 619 !4 = !{!5, !"cold"}
 620 !5 = !{i64 1, i64 2, i64 3, i64 5}
 621 !6 = !{i64 1}
 622 !7 = !{!8, !"hot"}
 623 !8 = !{i64 1, i64 2, i64 3, i64 6}
 624 )IR");
 625
 626   Function *Func = M->getFunction("test");
 627   CallBase *Call = findCall(*Func, "call");
 628
 629   CallStack<MDNode, MDNode::op_iterator> InstCallsite(
 630       Call->getMetadata(LLVMContext::MD_callsite));
 631
 632   MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);
 633   unsigned Idx = 0;
 634   for (auto &MIBOp : MemProfMD->operands()) {
 635     auto *MIBMD = cast<const MDNode>(MIBOp);
 636     MDNode *StackNode = getMIBStackNode(MIBMD);
 637     CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);
 638     EXPECT_EQ(StackContext.back(), 4 + Idx);
 639     std::vector<uint64_t> StackIds;
 640     for (auto ContextIter = StackContext.beginAfterSharedPrefix(InstCallsite);
 641          ContextIter != StackContext.end(); ++ContextIter)
 642       StackIds.push_back(*ContextIter);
 643     if (Idx == 0) {
 644       std::vector<uint64_t> Expected = {2, 3, 4};
 645       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 646     } else if (Idx == 1) {
 647       std::vector<uint64_t> Expected = {2, 3, 5};
 648       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 649     } else {
 650       std::vector<uint64_t> Expected = {2, 3, 6};
 651       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 652     }
 653     Idx++;
 654   }
 655 }
 656
 657 TEST_F(MemoryProfileInfoTest, CallStackTestSummary) {
 658   std::unique_ptr<ModuleSummaryIndex> Index = makeLLVMIndex(R"Summary(
 659 ^0 = module: (path: "test.o", hash: (0, 0, 0, 0, 0))
 660 ^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (1, 2, 3, 4)), (type: cold, stackIds: (1, 2, 3, 5)), (type: hot, stackIds: (1, 2, 3, 6))))))))
 661 ^2 = gv: (guid: 25, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1)), callsites: ((callee: ^1, clones: (0), stackIds: (3, 4)), (callee: ^1, clones: (0), stackIds: (3, 5)), (callee: ^1, clones: (0), stackIds: (3, 6))))))
 662 )Summary");
 663
 664   ASSERT_NE(Index, nullptr);
 665   auto *CallsiteSummary =
 666       cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/25));
 667   unsigned Idx = 0;
 668   for (auto &CI : CallsiteSummary->callsites()) {
 669     CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator> InstCallsite(
 670         &CI);
 671     std::vector<uint64_t> StackIds;
 672     for (auto StackIdIndex : InstCallsite)
 673       StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));
 674     if (Idx == 0) {
 675       std::vector<uint64_t> Expected = {3, 4};
 676       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 677     } else if (Idx == 1) {
 678       std::vector<uint64_t> Expected = {3, 5};
 679       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 680     } else {
 681       std::vector<uint64_t> Expected = {3, 6};
 682       EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 683     }
 684     Idx++;
 685   }
 686
 687   auto *AllocSummary =
 688       cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/23));
 689   for (auto &AI : AllocSummary->allocs()) {
 690     unsigned Idx = 0;
 691     for (auto &MIB : AI.MIBs) {
 692       CallStack<MIBInfo, SmallVector<unsigned>::const_iterator> StackContext(
 693           &MIB);
 694       EXPECT_EQ(Index->getStackIdAtIndex(StackContext.back()), 4 + Idx);
 695       std::vector<uint64_t> StackIds;
 696       for (auto StackIdIndex : StackContext)
 697         StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));
 698       if (Idx == 0) {
 699         std::vector<uint64_t> Expected = {1, 2, 3, 4};
 700         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 701       } else if (Idx == 1) {
 702         std::vector<uint64_t> Expected = {1, 2, 3, 5};
 703         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 704       } else {
 705         std::vector<uint64_t> Expected = {1, 2, 3, 6};
 706         EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));
 707       }
 708       Idx++;
 709     }
 710   }
 711 }
 712 } // end anonymous namespace