llvm/lib/Transforms/Instrumentation/MemProfiler.cpp

   1 //===- MemProfiler.cpp - memory allocation and access profiler ------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of MemProfiler. Memory accesses are instrumented
  10 // to increment the access count held in a shadow memory location, or
  11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
  12 // memcpy, memset) are changed to call the memory profiling runtime version
  13 // instead.
  14 //
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/ADT/Statistic.h"
  20 #include "llvm/ADT/StringRef.h"
  21 #include "llvm/ADT/Triple.h"
  22 #include "llvm/IR/Constant.h"
  23 #include "llvm/IR/DataLayout.h"
  24 #include "llvm/IR/Function.h"
  25 #include "llvm/IR/GlobalValue.h"
  26 #include "llvm/IR/IRBuilder.h"
  27 #include "llvm/IR/Instruction.h"
  28 #include "llvm/IR/LLVMContext.h"
  29 #include "llvm/IR/Module.h"
  30 #include "llvm/IR/Type.h"
  31 #include "llvm/IR/Value.h"
  32 #include "llvm/InitializePasses.h"
  33 #include "llvm/Pass.h"
  34 #include "llvm/Support/CommandLine.h"
  35 #include "llvm/Support/Debug.h"
  36 #include "llvm/Transforms/Instrumentation.h"
  37 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  38 #include "llvm/Transforms/Utils/ModuleUtils.h"
  39
  40 using namespace llvm;
  41
  42 #define DEBUG_TYPE "memprof"
  43
  44 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
  45
  46 // Size of memory mapped to a single shadow location.
  47 constexpr uint64_t DefaultShadowGranularity = 64;
  48
  49 // Scale from granularity down to shadow size.
  50 constexpr uint64_t DefaultShadowScale = 3;
  51
  52 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
  53 constexpr uint64_t MemProfCtorAndDtorPriority = 1;
  54 // On Emscripten, the system needs more than one priorities for constructors.
  55 constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
  56 constexpr char MemProfInitName[] = "__memprof_init";
  57 constexpr char MemProfVersionCheckNamePrefix[] =
  58     "__memprof_version_mismatch_check_v";
  59
  60 constexpr char MemProfShadowMemoryDynamicAddress[] =
  61     "__memprof_shadow_memory_dynamic_address";
  62
  63 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
  64
  65 // Command-line flags.
  66
  67 static cl::opt<bool> ClInsertVersionCheck(
  68     "memprof-guard-against-version-mismatch",
  69     cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
  70     cl::init(true));
  71
  72 // This flag may need to be replaced with -f[no-]memprof-reads.
  73 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
  74                                        cl::desc("instrument read instructions"),
  75                                        cl::Hidden, cl::init(true));
  76
  77 static cl::opt<bool>
  78     ClInstrumentWrites("memprof-instrument-writes",
  79                        cl::desc("instrument write instructions"), cl::Hidden,
  80                        cl::init(true));
  81
  82 static cl::opt<bool> ClInstrumentAtomics(
  83     "memprof-instrument-atomics",
  84     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
  85     cl::init(true));
  86
  87 static cl::opt<bool> ClUseCalls(
  88     "memprof-use-callbacks",
  89     cl::desc("Use callbacks instead of inline instrumentation sequences."),
  90     cl::Hidden, cl::init(false));
  91
  92 static cl::opt<std::string>
  93     ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
  94                                  cl::desc("Prefix for memory access callbacks"),
  95                                  cl::Hidden, cl::init("__memprof_"));
  96
  97 // These flags allow to change the shadow mapping.
  98 // The shadow mapping looks like
  99 //    Shadow = ((Mem & mask) >> scale) + offset
 100
 101 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
 102                                    cl::desc("scale of memprof shadow mapping"),
 103                                    cl::Hidden, cl::init(DefaultShadowScale));
 104
 105 static cl::opt<int>
 106     ClMappingGranularity("memprof-mapping-granularity",
 107                          cl::desc("granularity of memprof shadow mapping"),
 108                          cl::Hidden, cl::init(DefaultShadowGranularity));
 109
 110 // Debug flags.
 111
 112 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
 113                             cl::init(0));
 114
 115 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
 116                                         cl::desc("Debug func"));
 117
 118 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
 119                                cl::Hidden, cl::init(-1));
 120
 121 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
 122                                cl::Hidden, cl::init(-1));
 123
 124 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
 125 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
 126
 127 namespace {
 128
 129 /// This struct defines the shadow mapping using the rule:
 130 ///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
 131 struct ShadowMapping {
 132   ShadowMapping() {
 133     Scale = ClMappingScale;
 134     Granularity = ClMappingGranularity;
 135     Mask = ~(Granularity - 1);
 136   }
 137
 138   int Scale;
 139   int Granularity;
 140   uint64_t Mask; // Computed as ~(Granularity-1)
 141 };
 142
 143 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
 144   return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
 145                                        : MemProfCtorAndDtorPriority;
 146 }
 147
 148 struct InterestingMemoryAccess {
 149   Value *Addr = nullptr;
 150   bool IsWrite;
 151   unsigned Alignment;
 152   uint64_t TypeSize;
 153   Value *MaybeMask = nullptr;
 154 };
 155
 156 /// Instrument the code in module to profile memory accesses.
 157 class MemProfiler {
 158 public:
 159   MemProfiler(Module &M) {
 160     C = &(M.getContext());
 161     LongSize = M.getDataLayout().getPointerSizeInBits();
 162     IntptrTy = Type::getIntNTy(*C, LongSize);
 163   }
 164
 165   /// If it is an interesting memory access, populate information
 166   /// about the access and return a InterestingMemoryAccess struct.
 167   /// Otherwise return None.
 168   Optional<InterestingMemoryAccess>
 169   isInterestingMemoryAccess(Instruction *I) const;
 170
 171   void instrumentMop(Instruction *I, const DataLayout &DL,
 172                      InterestingMemoryAccess &Access);
 173   void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
 174                          Value *Addr, uint32_t TypeSize, bool IsWrite);
 175   void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
 176                                    Instruction *I, Value *Addr,
 177                                    unsigned Alignment, uint32_t TypeSize,
 178                                    bool IsWrite);
 179   void instrumentMemIntrinsic(MemIntrinsic *MI);
 180   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
 181   bool instrumentFunction(Function &F);
 182   bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
 183   bool insertDynamicShadowAtFunctionEntry(Function &F);
 184
 185 private:
 186   void initializeCallbacks(Module &M);
 187
 188   LLVMContext *C;
 189   int LongSize;
 190   Type *IntptrTy;
 191   ShadowMapping Mapping;
 192
 193   // These arrays is indexed by AccessIsWrite
 194   FunctionCallee MemProfMemoryAccessCallback[2];
 195   FunctionCallee MemProfMemoryAccessCallbackSized[2];
 196
 197   FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
 198   Value *DynamicShadowOffset = nullptr;
 199 };
 200
 201 class MemProfilerLegacyPass : public FunctionPass {
 202 public:
 203   static char ID;
 204
 205   explicit MemProfilerLegacyPass() : FunctionPass(ID) {
 206     initializeMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
 207   }
 208
 209   StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
 210
 211   bool runOnFunction(Function &F) override {
 212     MemProfiler Profiler(*F.getParent());
 213     return Profiler.instrumentFunction(F);
 214   }
 215 };
 216
 217 class ModuleMemProfiler {
 218 public:
 219   ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
 220
 221   bool instrumentModule(Module &);
 222
 223 private:
 224   Triple TargetTriple;
 225   ShadowMapping Mapping;
 226   Function *MemProfCtorFunction = nullptr;
 227 };
 228
 229 class ModuleMemProfilerLegacyPass : public ModulePass {
 230 public:
 231   static char ID;
 232
 233   explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
 234     initializeModuleMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
 235   }
 236
 237   StringRef getPassName() const override { return "ModuleMemProfiler"; }
 238
 239   void getAnalysisUsage(AnalysisUsage &AU) const override {}
 240
 241   bool runOnModule(Module &M) override {
 242     ModuleMemProfiler MemProfiler(M);
 243     return MemProfiler.instrumentModule(M);
 244   }
 245 };
 246
 247 } // end anonymous namespace
 248
 249 MemProfilerPass::MemProfilerPass() {}
 250
 251 PreservedAnalyses MemProfilerPass::run(Function &F,
 252                                        AnalysisManager<Function> &AM) {
 253   Module &M = *F.getParent();
 254   MemProfiler Profiler(M);
 255   if (Profiler.instrumentFunction(F))
 256     return PreservedAnalyses::none();
 257   return PreservedAnalyses::all();
 258 }
 259
 260 ModuleMemProfilerPass::ModuleMemProfilerPass() {}
 261
 262 PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
 263                                              AnalysisManager<Module> &AM) {
 264   ModuleMemProfiler Profiler(M);
 265   if (Profiler.instrumentModule(M))
 266     return PreservedAnalyses::none();
 267   return PreservedAnalyses::all();
 268 }
 269
 270 char MemProfilerLegacyPass::ID = 0;
 271
 272 INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
 273                       "MemProfiler: profile memory allocations and accesses.",
 274                       false, false)
 275 INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
 276                     "MemProfiler: profile memory allocations and accesses.",
 277                     false, false)
 278
 279 FunctionPass *llvm::createMemProfilerFunctionPass() {
 280   return new MemProfilerLegacyPass();
 281 }
 282
 283 char ModuleMemProfilerLegacyPass::ID = 0;
 284
 285 INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
 286                 "MemProfiler: profile memory allocations and accesses."
 287                 "ModulePass",
 288                 false, false)
 289
 290 ModulePass *llvm::createModuleMemProfilerLegacyPassPass() {
 291   return new ModuleMemProfilerLegacyPass();
 292 }
 293
 294 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
 295   // (Shadow & mask) >> scale
 296   Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
 297   Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
 298   // (Shadow >> scale) | offset
 299   assert(DynamicShadowOffset);
 300   return IRB.CreateAdd(Shadow, DynamicShadowOffset);
 301 }
 302
 303 // Instrument memset/memmove/memcpy
 304 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
 305   IRBuilder<> IRB(MI);
 306   if (isa<MemTransferInst>(MI)) {
 307     IRB.CreateCall(
 308         isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
 309         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 310          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
 311          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 312   } else if (isa<MemSetInst>(MI)) {
 313     IRB.CreateCall(
 314         MemProfMemset,
 315         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 316          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
 317          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 318   }
 319   MI->eraseFromParent();
 320 }
 321
 322 Optional<InterestingMemoryAccess>
 323 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
 324   // Do not instrument the load fetching the dynamic shadow address.
 325   if (DynamicShadowOffset == I)
 326     return None;
 327
 328   InterestingMemoryAccess Access;
 329
 330   const DataLayout &DL = I->getModule()->getDataLayout();
 331   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 332     if (!ClInstrumentReads)
 333       return None;
 334     Access.IsWrite = false;
 335     Access.TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
 336     Access.Alignment = LI->getAlignment();
 337     Access.Addr = LI->getPointerOperand();
 338   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 339     if (!ClInstrumentWrites)
 340       return None;
 341     Access.IsWrite = true;
 342     Access.TypeSize =
 343         DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
 344     Access.Alignment = SI->getAlignment();
 345     Access.Addr = SI->getPointerOperand();
 346   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
 347     if (!ClInstrumentAtomics)
 348       return None;
 349     Access.IsWrite = true;
 350     Access.TypeSize =
 351         DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
 352     Access.Alignment = 0;
 353     Access.Addr = RMW->getPointerOperand();
 354   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
 355     if (!ClInstrumentAtomics)
 356       return None;
 357     Access.IsWrite = true;
 358     Access.TypeSize =
 359         DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
 360     Access.Alignment = 0;
 361     Access.Addr = XCHG->getPointerOperand();
 362   } else if (auto *CI = dyn_cast<CallInst>(I)) {
 363     auto *F = CI->getCalledFunction();
 364     if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
 365               F->getIntrinsicID() == Intrinsic::masked_store)) {
 366       unsigned OpOffset = 0;
 367       if (F->getIntrinsicID() == Intrinsic::masked_store) {
 368         if (!ClInstrumentWrites)
 369           return None;
 370         // Masked store has an initial operand for the value.
 371         OpOffset = 1;
 372         Access.IsWrite = true;
 373       } else {
 374         if (!ClInstrumentReads)
 375           return None;
 376         Access.IsWrite = false;
 377       }
 378
 379       auto *BasePtr = CI->getOperand(0 + OpOffset);
 380       auto *Ty = cast<PointerType>(BasePtr->getType())->getElementType();
 381       Access.TypeSize = DL.getTypeStoreSizeInBits(Ty);
 382       if (auto *AlignmentConstant =
 383               dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
 384         Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
 385       else
 386         Access.Alignment = 1; // No alignment guarantees. We probably got Undef
 387       Access.MaybeMask = CI->getOperand(2 + OpOffset);
 388       Access.Addr = BasePtr;
 389     }
 390   }
 391
 392   if (!Access.Addr)
 393     return None;
 394
 395   // Do not instrument acesses from different address spaces; we cannot deal
 396   // with them.
 397   Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
 398   if (PtrTy->getPointerAddressSpace() != 0)
 399     return None;
 400
 401   // Ignore swifterror addresses.
 402   // swifterror memory addresses are mem2reg promoted by instruction
 403   // selection. As such they cannot have regular uses like an instrumentation
 404   // function and it makes no sense to track them as memory.
 405   if (Access.Addr->isSwiftError())
 406     return None;
 407
 408   return Access;
 409 }
 410
 411 void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
 412                                               Instruction *I, Value *Addr,
 413                                               unsigned Alignment,
 414                                               uint32_t TypeSize, bool IsWrite) {
 415   auto *VTy = cast<FixedVectorType>(
 416       cast<PointerType>(Addr->getType())->getElementType());
 417   uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
 418   unsigned Num = VTy->getNumElements();
 419   auto *Zero = ConstantInt::get(IntptrTy, 0);
 420   for (unsigned Idx = 0; Idx < Num; ++Idx) {
 421     Value *InstrumentedAddress = nullptr;
 422     Instruction *InsertBefore = I;
 423     if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
 424       // dyn_cast as we might get UndefValue
 425       if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
 426         if (Masked->isZero())
 427           // Mask is constant false, so no instrumentation needed.
 428           continue;
 429         // If we have a true or undef value, fall through to instrumentAddress.
 430         // with InsertBefore == I
 431       }
 432     } else {
 433       IRBuilder<> IRB(I);
 434       Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
 435       Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
 436       InsertBefore = ThenTerm;
 437     }
 438
 439     IRBuilder<> IRB(InsertBefore);
 440     InstrumentedAddress =
 441         IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
 442     instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
 443                       IsWrite);
 444   }
 445 }
 446
 447 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
 448                                 InterestingMemoryAccess &Access) {
 449   if (Access.IsWrite)
 450     NumInstrumentedWrites++;
 451   else
 452     NumInstrumentedReads++;
 453
 454   if (Access.MaybeMask) {
 455     instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
 456                                 Access.Alignment, Access.TypeSize,
 457                                 Access.IsWrite);
 458   } else {
 459     // Since the access counts will be accumulated across the entire allocation,
 460     // we only update the shadow access count for the first location and thus
 461     // don't need to worry about alignment and type size.
 462     instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
 463   }
 464 }
 465
 466 void MemProfiler::instrumentAddress(Instruction *OrigIns,
 467                                     Instruction *InsertBefore, Value *Addr,
 468                                     uint32_t TypeSize, bool IsWrite) {
 469   IRBuilder<> IRB(InsertBefore);
 470   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
 471
 472   if (ClUseCalls) {
 473     IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
 474     return;
 475   }
 476
 477   // Create an inline sequence to compute shadow location, and increment the
 478   // value by one.
 479   Type *ShadowTy = Type::getInt64Ty(*C);
 480   Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
 481   Value *ShadowPtr = memToShadow(AddrLong, IRB);
 482   Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
 483   Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
 484   Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
 485   ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
 486   IRB.CreateStore(ShadowValue, ShadowAddr);
 487 }
 488
 489 // Create the variable for the profile file name.
 490 void createProfileFileNameVar(Module &M) {
 491   const MDString *MemProfFilename =
 492       dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
 493   if (!MemProfFilename)
 494     return;
 495   assert(!MemProfFilename->getString().empty() &&
 496          "Unexpected MemProfProfileFilename metadata with empty string");
 497   Constant *ProfileNameConst = ConstantDataArray::getString(
 498       M.getContext(), MemProfFilename->getString(), true);
 499   GlobalVariable *ProfileNameVar = new GlobalVariable(
 500       M, ProfileNameConst->getType(), /*isConstant=*/true,
 501       GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
 502   Triple TT(M.getTargetTriple());
 503   if (TT.supportsCOMDAT()) {
 504     ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
 505     ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
 506   }
 507 }
 508
 509 bool ModuleMemProfiler::instrumentModule(Module &M) {
 510   // Create a module constructor.
 511   std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
 512   std::string VersionCheckName =
 513       ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
 514                            : "";
 515   std::tie(MemProfCtorFunction, std::ignore) =
 516       createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
 517                                           MemProfInitName, /*InitArgTypes=*/{},
 518                                           /*InitArgs=*/{}, VersionCheckName);
 519
 520   const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
 521   appendToGlobalCtors(M, MemProfCtorFunction, Priority);
 522
 523   createProfileFileNameVar(M);
 524
 525   return true;
 526 }
 527
 528 void MemProfiler::initializeCallbacks(Module &M) {
 529   IRBuilder<> IRB(*C);
 530
 531   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
 532     const std::string TypeStr = AccessIsWrite ? "store" : "load";
 533
 534     SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
 535     SmallVector<Type *, 2> Args1{1, IntptrTy};
 536     MemProfMemoryAccessCallbackSized[AccessIsWrite] =
 537         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
 538                               FunctionType::get(IRB.getVoidTy(), Args2, false));
 539
 540     MemProfMemoryAccessCallback[AccessIsWrite] =
 541         M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
 542                               FunctionType::get(IRB.getVoidTy(), Args1, false));
 543   }
 544   MemProfMemmove = M.getOrInsertFunction(
 545       ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
 546       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
 547   MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
 548                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 549                                         IRB.getInt8PtrTy(), IntptrTy);
 550   MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
 551                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 552                                         IRB.getInt32Ty(), IntptrTy);
 553 }
 554
 555 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
 556   // For each NSObject descendant having a +load method, this method is invoked
 557   // by the ObjC runtime before any of the static constructors is called.
 558   // Therefore we need to instrument such methods with a call to __memprof_init
 559   // at the beginning in order to initialize our runtime before any access to
 560   // the shadow memory.
 561   // We cannot just ignore these methods, because they may call other
 562   // instrumented functions.
 563   if (F.getName().find(" load]") != std::string::npos) {
 564     FunctionCallee MemProfInitFunction =
 565         declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
 566     IRBuilder<> IRB(&F.front(), F.front().begin());
 567     IRB.CreateCall(MemProfInitFunction, {});
 568     return true;
 569   }
 570   return false;
 571 }
 572
 573 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
 574   IRBuilder<> IRB(&F.front().front());
 575   Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
 576       MemProfShadowMemoryDynamicAddress, IntptrTy);
 577   if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
 578     cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
 579   DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
 580   return true;
 581 }
 582
 583 bool MemProfiler::instrumentFunction(Function &F) {
 584   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
 585     return false;
 586   if (ClDebugFunc == F.getName())
 587     return false;
 588   if (F.getName().startswith("__memprof_"))
 589     return false;
 590
 591   bool FunctionModified = false;
 592
 593   // If needed, insert __memprof_init.
 594   // This function needs to be called even if the function body is not
 595   // instrumented.
 596   if (maybeInsertMemProfInitAtFunctionEntry(F))
 597     FunctionModified = true;
 598
 599   LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
 600
 601   initializeCallbacks(*F.getParent());
 602
 603   FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
 604
 605   SmallVector<Instruction *, 16> ToInstrument;
 606
 607   // Fill the set of memory operations to instrument.
 608   for (auto &BB : F) {
 609     for (auto &Inst : BB) {
 610       if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
 611         ToInstrument.push_back(&Inst);
 612     }
 613   }
 614
 615   int NumInstrumented = 0;
 616   for (auto *Inst : ToInstrument) {
 617     if (ClDebugMin < 0 || ClDebugMax < 0 ||
 618         (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
 619       Optional<InterestingMemoryAccess> Access =
 620           isInterestingMemoryAccess(Inst);
 621       if (Access)
 622         instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
 623       else
 624         instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
 625     }
 626     NumInstrumented++;
 627   }
 628
 629   if (NumInstrumented > 0)
 630     FunctionModified = true;
 631
 632   LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
 633                     << F << "\n");
 634
 635   return FunctionModified;
 636 }