lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

   1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This file is a part of HWAddressSanitizer, an address sanity checker
  11 /// based on tagged addressing.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  15 #include "llvm/ADT/MapVector.h"
  16 #include "llvm/ADT/SmallVector.h"
  17 #include "llvm/ADT/StringExtras.h"
  18 #include "llvm/ADT/StringRef.h"
  19 #include "llvm/ADT/Triple.h"
  20 #include "llvm/BinaryFormat/ELF.h"
  21 #include "llvm/IR/Attributes.h"
  22 #include "llvm/IR/BasicBlock.h"
  23 #include "llvm/IR/Constant.h"
  24 #include "llvm/IR/Constants.h"
  25 #include "llvm/IR/DataLayout.h"
  26 #include "llvm/IR/DebugInfoMetadata.h"
  27 #include "llvm/IR/DerivedTypes.h"
  28 #include "llvm/IR/Function.h"
  29 #include "llvm/IR/IRBuilder.h"
  30 #include "llvm/IR/InlineAsm.h"
  31 #include "llvm/IR/InstVisitor.h"
  32 #include "llvm/IR/Instruction.h"
  33 #include "llvm/IR/Instructions.h"
  34 #include "llvm/IR/IntrinsicInst.h"
  35 #include "llvm/IR/Intrinsics.h"
  36 #include "llvm/IR/LLVMContext.h"
  37 #include "llvm/IR/MDBuilder.h"
  38 #include "llvm/IR/Module.h"
  39 #include "llvm/IR/Type.h"
  40 #include "llvm/IR/Value.h"
  41 #include "llvm/Pass.h"
  42 #include "llvm/Support/Casting.h"
  43 #include "llvm/Support/CommandLine.h"
  44 #include "llvm/Support/Debug.h"
  45 #include "llvm/Support/raw_ostream.h"
  46 #include "llvm/Transforms/Instrumentation.h"
  47 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  48 #include "llvm/Transforms/Utils/ModuleUtils.h"
  49 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
  50 #include <sstream>
  51
  52 using namespace llvm;
  53
  54 #define DEBUG_TYPE "hwasan"
  55
  56 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
  57 static const char *const kHwasanNoteName = "hwasan.note";
  58 static const char *const kHwasanInitName = "__hwasan_init";
  59 static const char *const kHwasanPersonalityThunkName =
  60     "__hwasan_personality_thunk";
  61
  62 static const char *const kHwasanShadowMemoryDynamicAddress =
  63     "__hwasan_shadow_memory_dynamic_address";
  64
  65 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
  66 static const size_t kNumberOfAccessSizes = 5;
  67
  68 static const size_t kDefaultShadowScale = 4;
  69 static const uint64_t kDynamicShadowSentinel =
  70     std::numeric_limits<uint64_t>::max();
  71 static const unsigned kPointerTagShift = 56;
  72
  73 static const unsigned kShadowBaseAlignment = 32;
  74
  75 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
  76     "hwasan-memory-access-callback-prefix",
  77     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
  78     cl::init("__hwasan_"));
  79
  80 static cl::opt<bool>
  81     ClInstrumentWithCalls("hwasan-instrument-with-calls",
  82                 cl::desc("instrument reads and writes with callbacks"),
  83                 cl::Hidden, cl::init(false));
  84
  85 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
  86                                        cl::desc("instrument read instructions"),
  87                                        cl::Hidden, cl::init(true));
  88
  89 static cl::opt<bool> ClInstrumentWrites(
  90     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
  91     cl::Hidden, cl::init(true));
  92
  93 static cl::opt<bool> ClInstrumentAtomics(
  94     "hwasan-instrument-atomics",
  95     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
  96     cl::init(true));
  97
  98 static cl::opt<bool> ClRecover(
  99     "hwasan-recover",
 100     cl::desc("Enable recovery mode (continue-after-error)."),
 101     cl::Hidden, cl::init(false));
 102
 103 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
 104                                        cl::desc("instrument stack (allocas)"),
 105                                        cl::Hidden, cl::init(true));
 106
 107 static cl::opt<bool> ClUARRetagToZero(
 108     "hwasan-uar-retag-to-zero",
 109     cl::desc("Clear alloca tags before returning from the function to allow "
 110              "non-instrumented and instrumented function calls mix. When set "
 111              "to false, allocas are retagged before returning from the "
 112              "function to detect use after return."),
 113     cl::Hidden, cl::init(true));
 114
 115 static cl::opt<bool> ClGenerateTagsWithCalls(
 116     "hwasan-generate-tags-with-calls",
 117     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
 118     cl::init(false));
 119
 120 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
 121                                cl::Hidden, cl::init(false));
 122
 123 static cl::opt<int> ClMatchAllTag(
 124     "hwasan-match-all-tag",
 125     cl::desc("don't report bad accesses via pointers with this tag"),
 126     cl::Hidden, cl::init(-1));
 127
 128 static cl::opt<bool> ClEnableKhwasan(
 129     "hwasan-kernel",
 130     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
 131     cl::Hidden, cl::init(false));
 132
 133 // These flags allow to change the shadow mapping and control how shadow memory
 134 // is accessed. The shadow mapping looks like:
 135 //    Shadow = (Mem >> scale) + offset
 136
 137 static cl::opt<uint64_t>
 138     ClMappingOffset("hwasan-mapping-offset",
 139                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
 140                     cl::Hidden, cl::init(0));
 141
 142 static cl::opt<bool>
 143     ClWithIfunc("hwasan-with-ifunc",
 144                 cl::desc("Access dynamic shadow through an ifunc global on "
 145                          "platforms that support this"),
 146                 cl::Hidden, cl::init(false));
 147
 148 static cl::opt<bool> ClWithTls(
 149     "hwasan-with-tls",
 150     cl::desc("Access dynamic shadow through an thread-local pointer on "
 151              "platforms that support this"),
 152     cl::Hidden, cl::init(true));
 153
 154 static cl::opt<bool>
 155     ClRecordStackHistory("hwasan-record-stack-history",
 156                          cl::desc("Record stack frames with tagged allocations "
 157                                   "in a thread-local ring buffer"),
 158                          cl::Hidden, cl::init(true));
 159 static cl::opt<bool>
 160     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
 161                               cl::desc("instrument memory intrinsics"),
 162                               cl::Hidden, cl::init(true));
 163
 164 static cl::opt<bool>
 165     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
 166                             cl::desc("instrument landing pads"), cl::Hidden,
 167                             cl::init(false), cl::ZeroOrMore);
 168
 169 static cl::opt<bool> ClUseShortGranules(
 170     "hwasan-use-short-granules",
 171     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
 172     cl::init(false), cl::ZeroOrMore);
 173
 174 static cl::opt<bool> ClInstrumentPersonalityFunctions(
 175     "hwasan-instrument-personality-functions",
 176     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
 177     cl::ZeroOrMore);
 178
 179 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
 180                                        cl::desc("inline all checks"),
 181                                        cl::Hidden, cl::init(false));
 182
 183 namespace {
 184
 185 /// An instrumentation pass implementing detection of addressability bugs
 186 /// using tagged pointers.
 187 class HWAddressSanitizer {
 188 public:
 189   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
 190                               bool Recover = false) : M(M) {
 191     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
 192     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
 193         ClEnableKhwasan : CompileKernel;
 194
 195     initializeModule();
 196   }
 197
 198   bool sanitizeFunction(Function &F);
 199   void initializeModule();
 200
 201   void initializeCallbacks(Module &M);
 202
 203   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
 204   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 205
 206   void untagPointerOperand(Instruction *I, Value *Addr);
 207   Value *shadowBase();
 208   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
 209   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 210                                  unsigned AccessSizeIndex,
 211                                  Instruction *InsertBefore);
 212   void instrumentMemIntrinsic(MemIntrinsic *MI);
 213   bool instrumentMemAccess(Instruction *I);
 214   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
 215                                    uint64_t *TypeSize, unsigned *Alignment,
 216                                    Value **MaybeMask);
 217
 218   bool isInterestingAlloca(const AllocaInst &AI);
 219   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
 220   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
 221   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
 222   bool instrumentStack(
 223       SmallVectorImpl<AllocaInst *> &Allocas,
 224       DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap,
 225       SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
 226   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
 227   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
 228   Value *getNextTagWithCall(IRBuilder<> &IRB);
 229   Value *getStackBaseTag(IRBuilder<> &IRB);
 230   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
 231                      unsigned AllocaNo);
 232   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 233
 234   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
 235   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
 236
 237   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
 238   void instrumentGlobals();
 239
 240   void instrumentPersonalityFunctions();
 241
 242 private:
 243   LLVMContext *C;
 244   Module &M;
 245   Triple TargetTriple;
 246   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
 247   FunctionCallee HWAsanHandleVfork;
 248
 249   /// This struct defines the shadow mapping using the rule:
 250   ///   shadow = (mem >> Scale) + Offset.
 251   /// If InGlobal is true, then
 252   ///   extern char __hwasan_shadow[];
 253   ///   shadow = (mem >> Scale) + &__hwasan_shadow
 254   /// If InTls is true, then
 255   ///   extern char *__hwasan_tls;
 256   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
 257   struct ShadowMapping {
 258     int Scale;
 259     uint64_t Offset;
 260     bool InGlobal;
 261     bool InTls;
 262
 263     void init(Triple &TargetTriple);
 264     unsigned getObjectAlignment() const { return 1U << Scale; }
 265   };
 266   ShadowMapping Mapping;
 267
 268   Type *VoidTy = Type::getVoidTy(M.getContext());
 269   Type *IntptrTy;
 270   Type *Int8PtrTy;
 271   Type *Int8Ty;
 272   Type *Int32Ty;
 273   Type *Int64Ty = Type::getInt64Ty(M.getContext());
 274
 275   bool CompileKernel;
 276   bool Recover;
 277   bool UseShortGranules;
 278   bool InstrumentLandingPads;
 279
 280   Function *HwasanCtorFunction;
 281
 282   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
 283   FunctionCallee HwasanMemoryAccessCallbackSized[2];
 284
 285   FunctionCallee HwasanTagMemoryFunc;
 286   FunctionCallee HwasanGenerateTagFunc;
 287   FunctionCallee HwasanThreadEnterFunc;
 288
 289   Constant *ShadowGlobal;
 290
 291   Value *LocalDynamicShadow = nullptr;
 292   Value *StackBaseTag = nullptr;
 293   GlobalValue *ThreadPtrGlobal = nullptr;
 294 };
 295
 296 class HWAddressSanitizerLegacyPass : public FunctionPass {
 297 public:
 298   // Pass identification, replacement for typeid.
 299   static char ID;
 300
 301   explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
 302                                         bool Recover = false)
 303       : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {}
 304
 305   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 306
 307   bool doInitialization(Module &M) override {
 308     HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
 309     return true;
 310   }
 311
 312   bool runOnFunction(Function &F) override {
 313     return HWASan->sanitizeFunction(F);
 314   }
 315
 316   bool doFinalization(Module &M) override {
 317     HWASan.reset();
 318     return false;
 319   }
 320
 321 private:
 322   std::unique_ptr<HWAddressSanitizer> HWASan;
 323   bool CompileKernel;
 324   bool Recover;
 325 };
 326
 327 } // end anonymous namespace
 328
 329 char HWAddressSanitizerLegacyPass::ID = 0;
 330
 331 INITIALIZE_PASS_BEGIN(
 332     HWAddressSanitizerLegacyPass, "hwasan",
 333     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 334     false)
 335 INITIALIZE_PASS_END(
 336     HWAddressSanitizerLegacyPass, "hwasan",
 337     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 338     false)
 339
 340 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
 341                                                            bool Recover) {
 342   assert(!CompileKernel || Recover);
 343   return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
 344 }
 345
 346 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
 347     : CompileKernel(CompileKernel), Recover(Recover) {}
 348
 349 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
 350                                               ModuleAnalysisManager &MAM) {
 351   HWAddressSanitizer HWASan(M, CompileKernel, Recover);
 352   bool Modified = false;
 353   for (Function &F : M)
 354     Modified |= HWASan.sanitizeFunction(F);
 355   if (Modified)
 356     return PreservedAnalyses::none();
 357   return PreservedAnalyses::all();
 358 }
 359
 360 /// Module-level initialization.
 361 ///
 362 /// inserts a call to __hwasan_init to the module's constructor list.
 363 void HWAddressSanitizer::initializeModule() {
 364   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
 365   auto &DL = M.getDataLayout();
 366
 367   TargetTriple = Triple(M.getTargetTriple());
 368
 369   Mapping.init(TargetTriple);
 370
 371   C = &(M.getContext());
 372   IRBuilder<> IRB(*C);
 373   IntptrTy = IRB.getIntPtrTy(DL);
 374   Int8PtrTy = IRB.getInt8PtrTy();
 375   Int8Ty = IRB.getInt8Ty();
 376   Int32Ty = IRB.getInt32Ty();
 377
 378   HwasanCtorFunction = nullptr;
 379
 380   // Older versions of Android do not have the required runtime support for
 381   // short granules, global or personality function instrumentation. On other
 382   // platforms we currently require using the latest version of the runtime.
 383   bool NewRuntime =
 384       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
 385
 386   UseShortGranules =
 387       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
 388
 389   // If we don't have personality function support, fall back to landing pads.
 390   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
 391                               ? ClInstrumentLandingPads
 392                               : !NewRuntime;
 393
 394   if (!CompileKernel) {
 395     std::tie(HwasanCtorFunction, std::ignore) =
 396         getOrCreateSanitizerCtorAndInitFunctions(
 397             M, kHwasanModuleCtorName, kHwasanInitName,
 398             /*InitArgTypes=*/{},
 399             /*InitArgs=*/{},
 400             // This callback is invoked when the functions are created the first
 401             // time. Hook them into the global ctors list in that case:
 402             [&](Function *Ctor, FunctionCallee) {
 403               Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
 404               Ctor->setComdat(CtorComdat);
 405               appendToGlobalCtors(M, Ctor, 0, Ctor);
 406             });
 407
 408     bool InstrumentGlobals =
 409         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
 410     if (InstrumentGlobals)
 411       instrumentGlobals();
 412
 413     bool InstrumentPersonalityFunctions =
 414         ClInstrumentPersonalityFunctions.getNumOccurrences()
 415             ? ClInstrumentPersonalityFunctions
 416             : NewRuntime;
 417     if (InstrumentPersonalityFunctions)
 418       instrumentPersonalityFunctions();
 419   }
 420
 421   if (!TargetTriple.isAndroid()) {
 422     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
 423       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
 424                                     GlobalValue::ExternalLinkage, nullptr,
 425                                     "__hwasan_tls", nullptr,
 426                                     GlobalVariable::InitialExecTLSModel);
 427       appendToCompilerUsed(M, GV);
 428       return GV;
 429     });
 430     ThreadPtrGlobal = cast<GlobalVariable>(C);
 431   }
 432 }
 433
 434 void HWAddressSanitizer::initializeCallbacks(Module &M) {
 435   IRBuilder<> IRB(*C);
 436   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
 437     const std::string TypeStr = AccessIsWrite ? "store" : "load";
 438     const std::string EndingStr = Recover ? "_noabort" : "";
 439
 440     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
 441         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
 442         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
 443
 444     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
 445          AccessSizeIndex++) {
 446       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
 447           M.getOrInsertFunction(
 448               ClMemoryAccessCallbackPrefix + TypeStr +
 449                   itostr(1ULL << AccessSizeIndex) + EndingStr,
 450               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
 451     }
 452   }
 453
 454   HwasanTagMemoryFunc = M.getOrInsertFunction(
 455       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
 456   HwasanGenerateTagFunc =
 457       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
 458
 459   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
 460                                      ArrayType::get(IRB.getInt8Ty(), 0));
 461
 462   const std::string MemIntrinCallbackPrefix =
 463       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
 464   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
 465                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 466                                         IRB.getInt8PtrTy(), IntptrTy);
 467   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
 468                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 469                                        IRB.getInt8PtrTy(), IntptrTy);
 470   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
 471                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 472                                        IRB.getInt32Ty(), IntptrTy);
 473
 474   HWAsanHandleVfork =
 475       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
 476
 477   HwasanThreadEnterFunc =
 478       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy());
 479 }
 480
 481 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
 482   // An empty inline asm with input reg == output reg.
 483   // An opaque no-op cast, basically.
 484   InlineAsm *Asm = InlineAsm::get(
 485       FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
 486       StringRef(""), StringRef("=r,0"),
 487       /*hasSideEffects=*/false);
 488   return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
 489 }
 490
 491 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
 492   // Generate code only when dynamic addressing is needed.
 493   if (Mapping.Offset != kDynamicShadowSentinel)
 494     return nullptr;
 495
 496   if (Mapping.InGlobal) {
 497     return getDynamicShadowIfunc(IRB);
 498   } else {
 499     Value *GlobalDynamicAddress =
 500         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
 501             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
 502     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
 503   }
 504 }
 505
 506 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
 507                                                      bool *IsWrite,
 508                                                      uint64_t *TypeSize,
 509                                                      unsigned *Alignment,
 510                                                      Value **MaybeMask) {
 511   // Skip memory accesses inserted by another instrumentation.
 512   if (I->hasMetadata("nosanitize")) return nullptr;
 513
 514   // Do not instrument the load fetching the dynamic shadow address.
 515   if (LocalDynamicShadow == I)
 516     return nullptr;
 517
 518   Value *PtrOperand = nullptr;
 519   const DataLayout &DL = I->getModule()->getDataLayout();
 520   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 521     if (!ClInstrumentReads) return nullptr;
 522     *IsWrite = false;
 523     *TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
 524     *Alignment = LI->getAlignment();
 525     PtrOperand = LI->getPointerOperand();
 526   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 527     if (!ClInstrumentWrites) return nullptr;
 528     *IsWrite = true;
 529     *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
 530     *Alignment = SI->getAlignment();
 531     PtrOperand = SI->getPointerOperand();
 532   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
 533     if (!ClInstrumentAtomics) return nullptr;
 534     *IsWrite = true;
 535     *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
 536     *Alignment = 0;
 537     PtrOperand = RMW->getPointerOperand();
 538   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
 539     if (!ClInstrumentAtomics) return nullptr;
 540     *IsWrite = true;
 541     *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
 542     *Alignment = 0;
 543     PtrOperand = XCHG->getPointerOperand();
 544   }
 545
 546   if (PtrOperand) {
 547     // Do not instrument accesses from different address spaces; we cannot deal
 548     // with them.
 549     Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
 550     if (PtrTy->getPointerAddressSpace() != 0)
 551       return nullptr;
 552
 553     // Ignore swifterror addresses.
 554     // swifterror memory addresses are mem2reg promoted by instruction
 555     // selection. As such they cannot have regular uses like an instrumentation
 556     // function and it makes no sense to track them as memory.
 557     if (PtrOperand->isSwiftError())
 558       return nullptr;
 559   }
 560
 561   return PtrOperand;
 562 }
 563
 564 static unsigned getPointerOperandIndex(Instruction *I) {
 565   if (LoadInst *LI = dyn_cast<LoadInst>(I))
 566     return LI->getPointerOperandIndex();
 567   if (StoreInst *SI = dyn_cast<StoreInst>(I))
 568     return SI->getPointerOperandIndex();
 569   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
 570     return RMW->getPointerOperandIndex();
 571   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
 572     return XCHG->getPointerOperandIndex();
 573   report_fatal_error("Unexpected instruction");
 574   return -1;
 575 }
 576
 577 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
 578   size_t Res = countTrailingZeros(TypeSize / 8);
 579   assert(Res < kNumberOfAccessSizes);
 580   return Res;
 581 }
 582
 583 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
 584   if (TargetTriple.isAArch64())
 585     return;
 586
 587   IRBuilder<> IRB(I);
 588   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
 589   Value *UntaggedPtr =
 590       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
 591   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 592 }
 593
 594 Value *HWAddressSanitizer::shadowBase() {
 595   if (LocalDynamicShadow)
 596     return LocalDynamicShadow;
 597   return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
 598                                    Int8PtrTy);
 599 }
 600
 601 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
 602   // Mem >> Scale
 603   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
 604   if (Mapping.Offset == 0)
 605     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
 606   // (Mem >> Scale) + Offset
 607   return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
 608 }
 609
 610 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 611                                                    unsigned AccessSizeIndex,
 612                                                    Instruction *InsertBefore) {
 613   const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
 614   IRBuilder<> IRB(InsertBefore);
 615
 616   if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
 617       TargetTriple.isOSBinFormatELF() && !Recover) {
 618     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 619     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
 620     IRB.CreateCall(Intrinsic::getDeclaration(
 621                        M, UseShortGranules
 622                               ? Intrinsic::hwasan_check_memaccess_shortgranules
 623                               : Intrinsic::hwasan_check_memaccess),
 624                    {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
 625     return;
 626   }
 627
 628   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
 629   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
 630                                   IRB.getInt8Ty());
 631   Value *AddrLong = untagPointer(IRB, PtrLong);
 632   Value *Shadow = memToShadow(AddrLong, IRB);
 633   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
 634   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 635
 636   int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
 637       ClMatchAllTag : (CompileKernel ? 0xFF : -1);
 638   if (matchAllTag != -1) {
 639     Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
 640         ConstantInt::get(PtrTag->getType(), matchAllTag));
 641     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
 642   }
 643
 644   Instruction *CheckTerm =
 645       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
 646                                 MDBuilder(*C).createBranchWeights(1, 100000));
 647
 648   IRB.SetInsertPoint(CheckTerm);
 649   Value *OutOfShortGranuleTagRange =
 650       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
 651   Instruction *CheckFailTerm =
 652       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
 653                                 MDBuilder(*C).createBranchWeights(1, 100000));
 654
 655   IRB.SetInsertPoint(CheckTerm);
 656   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
 657   PtrLowBits = IRB.CreateAdd(
 658       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
 659   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
 660   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
 661                             MDBuilder(*C).createBranchWeights(1, 100000),
 662                             nullptr, nullptr, CheckFailTerm->getParent());
 663
 664   IRB.SetInsertPoint(CheckTerm);
 665   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
 666   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
 667   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
 668   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
 669   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
 670                             MDBuilder(*C).createBranchWeights(1, 100000),
 671                             nullptr, nullptr, CheckFailTerm->getParent());
 672
 673   IRB.SetInsertPoint(CheckFailTerm);
 674   InlineAsm *Asm;
 675   switch (TargetTriple.getArch()) {
 676     case Triple::x86_64:
 677       // The signal handler will find the data address in rdi.
 678       Asm = InlineAsm::get(
 679           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 680           "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
 681           "{rdi}",
 682           /*hasSideEffects=*/true);
 683       break;
 684     case Triple::aarch64:
 685     case Triple::aarch64_be:
 686       // The signal handler will find the data address in x0.
 687       Asm = InlineAsm::get(
 688           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 689           "brk #" + itostr(0x900 + AccessInfo),
 690           "{x0}",
 691           /*hasSideEffects=*/true);
 692       break;
 693     default:
 694       report_fatal_error("unsupported architecture");
 695   }
 696   IRB.CreateCall(Asm, PtrLong);
 697   if (Recover)
 698     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
 699 }
 700
 701 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
 702   IRBuilder<> IRB(MI);
 703   if (isa<MemTransferInst>(MI)) {
 704     IRB.CreateCall(
 705         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
 706         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 707          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
 708          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 709   } else if (isa<MemSetInst>(MI)) {
 710     IRB.CreateCall(
 711         HWAsanMemset,
 712         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 713          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
 714          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 715   }
 716   MI->eraseFromParent();
 717 }
 718
 719 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
 720   LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
 721   bool IsWrite = false;
 722   unsigned Alignment = 0;
 723   uint64_t TypeSize = 0;
 724   Value *MaybeMask = nullptr;
 725
 726   if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) {
 727     instrumentMemIntrinsic(cast<MemIntrinsic>(I));
 728     return true;
 729   }
 730
 731   Value *Addr =
 732       isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
 733
 734   if (!Addr)
 735     return false;
 736
 737   if (MaybeMask)
 738     return false; //FIXME
 739
 740   IRBuilder<> IRB(I);
 741   if (isPowerOf2_64(TypeSize) &&
 742       (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
 743       (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
 744        Alignment >= TypeSize / 8)) {
 745     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
 746     if (ClInstrumentWithCalls) {
 747       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
 748                      IRB.CreatePointerCast(Addr, IntptrTy));
 749     } else {
 750       instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I);
 751     }
 752   } else {
 753     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
 754                    {IRB.CreatePointerCast(Addr, IntptrTy),
 755                     ConstantInt::get(IntptrTy, TypeSize / 8)});
 756   }
 757   untagPointerOperand(I, Addr);
 758
 759   return true;
 760 }
 761
 762 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
 763   uint64_t ArraySize = 1;
 764   if (AI.isArrayAllocation()) {
 765     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
 766     assert(CI && "non-constant array size");
 767     ArraySize = CI->getZExtValue();
 768   }
 769   Type *Ty = AI.getAllocatedType();
 770   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
 771   return SizeInBytes * ArraySize;
 772 }
 773
 774 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
 775                                    Value *Tag, size_t Size) {
 776   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
 777   if (!UseShortGranules)
 778     Size = AlignedSize;
 779
 780   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
 781   if (ClInstrumentWithCalls) {
 782     IRB.CreateCall(HwasanTagMemoryFunc,
 783                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
 784                     ConstantInt::get(IntptrTy, AlignedSize)});
 785   } else {
 786     size_t ShadowSize = Size >> Mapping.Scale;
 787     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
 788     // If this memset is not inlined, it will be intercepted in the hwasan
 789     // runtime library. That's OK, because the interceptor skips the checks if
 790     // the address is in the shadow region.
 791     // FIXME: the interceptor is not as fast as real memset. Consider lowering
 792     // llvm.memset right here into either a sequence of stores, or a call to
 793     // hwasan_tag_memory.
 794     if (ShadowSize)
 795       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
 796     if (Size != AlignedSize) {
 797       IRB.CreateStore(
 798           ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
 799           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
 800       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
 801                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
 802                                    AlignedSize - 1));
 803     }
 804   }
 805   return true;
 806 }
 807
 808 static unsigned RetagMask(unsigned AllocaNo) {
 809   // A list of 8-bit numbers that have at most one run of non-zero bits.
 810   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
 811   // masks.
 812   // The list does not include the value 255, which is used for UAR.
 813   //
 814   // Because we are more likely to use earlier elements of this list than later
 815   // ones, it is sorted in increasing order of probability of collision with a
 816   // mask allocated (temporally) nearby. The program that generated this list
 817   // can be found at:
 818   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
 819   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
 820                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
 821                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
 822                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
 823   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
 824 }
 825
 826 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
 827   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
 828 }
 829
 830 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
 831   if (ClGenerateTagsWithCalls)
 832     return getNextTagWithCall(IRB);
 833   if (StackBaseTag)
 834     return StackBaseTag;
 835   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
 836   // first).
 837   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 838   auto GetStackPointerFn = Intrinsic::getDeclaration(
 839       M, Intrinsic::frameaddress,
 840       IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
 841   Value *StackPointer = IRB.CreateCall(
 842       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
 843
 844   // Extract some entropy from the stack pointer for the tags.
 845   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
 846   // between functions).
 847   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
 848   Value *StackTag =
 849       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
 850                     "hwasan.stack.base.tag");
 851   return StackTag;
 852 }
 853
 854 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
 855                                         AllocaInst *AI, unsigned AllocaNo) {
 856   if (ClGenerateTagsWithCalls)
 857     return getNextTagWithCall(IRB);
 858   return IRB.CreateXor(StackTag,
 859                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
 860 }
 861
 862 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
 863   if (ClUARRetagToZero)
 864     return ConstantInt::get(IntptrTy, 0);
 865   if (ClGenerateTagsWithCalls)
 866     return getNextTagWithCall(IRB);
 867   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
 868 }
 869
 870 // Add a tag to an address.
 871 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
 872                                       Value *PtrLong, Value *Tag) {
 873   Value *TaggedPtrLong;
 874   if (CompileKernel) {
 875     // Kernel addresses have 0xFF in the most significant byte.
 876     Value *ShiftedTag = IRB.CreateOr(
 877         IRB.CreateShl(Tag, kPointerTagShift),
 878         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
 879     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
 880   } else {
 881     // Userspace can simply do OR (tag << 56);
 882     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
 883     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
 884   }
 885   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
 886 }
 887
 888 // Remove tag from an address.
 889 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
 890   Value *UntaggedPtrLong;
 891   if (CompileKernel) {
 892     // Kernel addresses have 0xFF in the most significant byte.
 893     UntaggedPtrLong = IRB.CreateOr(PtrLong,
 894         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
 895   } else {
 896     // Userspace addresses have 0x00.
 897     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
 898         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
 899   }
 900   return UntaggedPtrLong;
 901 }
 902
 903 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
 904   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 905   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
 906     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
 907     // in Bionic's libc/private/bionic_tls.h.
 908     Function *ThreadPointerFunc =
 909         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
 910     Value *SlotPtr = IRB.CreatePointerCast(
 911         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
 912                                IRB.CreateCall(ThreadPointerFunc), 0x30),
 913         Ty->getPointerTo(0));
 914     return SlotPtr;
 915   }
 916   if (ThreadPtrGlobal)
 917     return ThreadPtrGlobal;
 918
 919
 920   return nullptr;
 921 }
 922
 923 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
 924   if (!Mapping.InTls) {
 925     LocalDynamicShadow = getDynamicShadowNonTls(IRB);
 926     return;
 927   }
 928
 929   if (!WithFrameRecord && TargetTriple.isAndroid()) {
 930     LocalDynamicShadow = getDynamicShadowIfunc(IRB);
 931     return;
 932   }
 933
 934   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
 935   assert(SlotPtr);
 936
 937   Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 938
 939   Function *F = IRB.GetInsertBlock()->getParent();
 940   if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
 941     Value *ThreadLongEqZero =
 942         IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
 943     auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
 944         ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
 945         false, MDBuilder(*C).createBranchWeights(1, 100000)));
 946
 947     IRB.SetInsertPoint(Br);
 948     // FIXME: This should call a new runtime function with a custom calling
 949     // convention to avoid needing to spill all arguments here.
 950     IRB.CreateCall(HwasanThreadEnterFunc);
 951     LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 952
 953     IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin());
 954     PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2);
 955     ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent());
 956     ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent());
 957     ThreadLong = ThreadLongPhi;
 958   }
 959
 960   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
 961   Value *ThreadLongMaybeUntagged =
 962       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
 963
 964   if (WithFrameRecord) {
 965     StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
 966
 967     // Prepare ring buffer data.
 968     Value *PC;
 969     if (TargetTriple.getArch() == Triple::aarch64)
 970       PC = readRegister(IRB, "pc");
 971     else
 972       PC = IRB.CreatePtrToInt(F, IntptrTy);
 973     Module *M = F->getParent();
 974     auto GetStackPointerFn = Intrinsic::getDeclaration(
 975         M, Intrinsic::frameaddress,
 976         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
 977     Value *SP = IRB.CreatePtrToInt(
 978         IRB.CreateCall(GetStackPointerFn,
 979                        {Constant::getNullValue(IRB.getInt32Ty())}),
 980         IntptrTy);
 981     // Mix SP and PC.
 982     // Assumptions:
 983     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
 984     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
 985     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
 986     //       0xSSSSPPPPPPPPPPPP
 987     SP = IRB.CreateShl(SP, 44);
 988
 989     // Store data to ring buffer.
 990     Value *RecordPtr =
 991         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
 992     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
 993
 994     // Update the ring buffer. Top byte of ThreadLong defines the size of the
 995     // buffer in pages, it must be a power of two, and the start of the buffer
 996     // must be aligned by twice that much. Therefore wrap around of the ring
 997     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
 998     // The use of AShr instead of LShr is due to
 999     //   https://bugs.llvm.org/show_bug.cgi?id=39030
1000     // Runtime library makes sure not to use the highest bit.
1001     Value *WrapMask = IRB.CreateXor(
1002         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1003         ConstantInt::get(IntptrTy, (uint64_t)-1));
1004     Value *ThreadLongNew = IRB.CreateAnd(
1005         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1006     IRB.CreateStore(ThreadLongNew, SlotPtr);
1007   }
1008
1009   // Get shadow base address by aligning RecordPtr up.
1010   // Note: this is not correct if the pointer is already aligned.
1011   // Runtime library will make sure this never happens.
1012   LocalDynamicShadow = IRB.CreateAdd(
1013       IRB.CreateOr(
1014           ThreadLongMaybeUntagged,
1015           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1016       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1017   LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy);
1018 }
1019
1020 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1021   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1022   Function *ReadRegister =
1023       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1024   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1025   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1026   return IRB.CreateCall(ReadRegister, Args);
1027 }
1028
1029 bool HWAddressSanitizer::instrumentLandingPads(
1030     SmallVectorImpl<Instruction *> &LandingPadVec) {
1031   for (auto *LP : LandingPadVec) {
1032     IRBuilder<> IRB(LP->getNextNode());
1033     IRB.CreateCall(
1034         HWAsanHandleVfork,
1035         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1036                                                                       : "sp")});
1037   }
1038   return true;
1039 }
1040
1041 bool HWAddressSanitizer::instrumentStack(
1042     SmallVectorImpl<AllocaInst *> &Allocas,
1043     DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap,
1044     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1045   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1046   // alloca addresses using that. Unfortunately, offsets are not known yet
1047   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1048   // temp, shift-OR it into each alloca address and xor with the retag mask.
1049   // This generates one extra instruction per alloca use.
1050   for (unsigned N = 0; N < Allocas.size(); ++N) {
1051     auto *AI = Allocas[N];
1052     IRBuilder<> IRB(AI->getNextNode());
1053
1054     // Replace uses of the alloca with tagged address.
1055     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1056     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1057     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1058     std::string Name =
1059         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1060     Replacement->setName(Name + ".hwasan");
1061
1062     AI->replaceUsesWithIf(Replacement,
1063                           [AILong](Use &U) { return U.getUser() != AILong; });
1064
1065     for (auto *DDI : AllocaDeclareMap.lookup(AI)) {
1066       DIExpression *OldExpr = DDI->getExpression();
1067       DIExpression *NewExpr = DIExpression::append(
1068           OldExpr, {dwarf::DW_OP_LLVM_tag_offset, RetagMask(N)});
1069       DDI->setArgOperand(2, MetadataAsValue::get(*C, NewExpr));
1070     }
1071
1072     size_t Size = getAllocaSizeInBytes(*AI);
1073     tagAlloca(IRB, AI, Tag, Size);
1074
1075     for (auto RI : RetVec) {
1076       IRB.SetInsertPoint(RI);
1077
1078       // Re-tag alloca memory with the special UAR tag.
1079       Value *Tag = getUARTag(IRB, StackTag);
1080       tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1081     }
1082   }
1083
1084   return true;
1085 }
1086
1087 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1088   return (AI.getAllocatedType()->isSized() &&
1089           // FIXME: instrument dynamic allocas, too
1090           AI.isStaticAlloca() &&
1091           // alloca() may be called with 0 size, ignore it.
1092           getAllocaSizeInBytes(AI) > 0 &&
1093           // We are only interested in allocas not promotable to registers.
1094           // Promotable allocas are common under -O0.
1095           !isAllocaPromotable(&AI) &&
1096           // inalloca allocas are not treated as static, and we don't want
1097           // dynamic alloca instrumentation for them as well.
1098           !AI.isUsedWithInAlloca() &&
1099           // swifterror allocas are register promoted by ISel
1100           !AI.isSwiftError());
1101 }
1102
1103 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1104   if (&F == HwasanCtorFunction)
1105     return false;
1106
1107   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1108     return false;
1109
1110   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1111
1112   SmallVector<Instruction*, 16> ToInstrument;
1113   SmallVector<AllocaInst*, 8> AllocasToInstrument;
1114   SmallVector<Instruction*, 8> RetVec;
1115   SmallVector<Instruction*, 8> LandingPadVec;
1116   DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> AllocaDeclareMap;
1117   for (auto &BB : F) {
1118     for (auto &Inst : BB) {
1119       if (ClInstrumentStack)
1120         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1121           if (isInterestingAlloca(*AI))
1122             AllocasToInstrument.push_back(AI);
1123           continue;
1124         }
1125
1126       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1127           isa<CleanupReturnInst>(Inst))
1128         RetVec.push_back(&Inst);
1129
1130       if (auto *DDI = dyn_cast<DbgDeclareInst>(&Inst))
1131         if (auto *Alloca = dyn_cast_or_null<AllocaInst>(DDI->getAddress()))
1132           AllocaDeclareMap[Alloca].push_back(DDI);
1133
1134       if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1135         LandingPadVec.push_back(&Inst);
1136
1137       Value *MaybeMask = nullptr;
1138       bool IsWrite;
1139       unsigned Alignment;
1140       uint64_t TypeSize;
1141       Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize,
1142                                               &Alignment, &MaybeMask);
1143       if (Addr || isa<MemIntrinsic>(Inst))
1144         ToInstrument.push_back(&Inst);
1145     }
1146   }
1147
1148   initializeCallbacks(*F.getParent());
1149
1150   if (!LandingPadVec.empty())
1151     instrumentLandingPads(LandingPadVec);
1152
1153   if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1154       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1155     // __hwasan_personality_thunk is a no-op for functions without an
1156     // instrumented stack, so we can drop it.
1157     F.setPersonalityFn(nullptr);
1158   }
1159
1160   if (AllocasToInstrument.empty() && ToInstrument.empty())
1161     return false;
1162
1163   assert(!LocalDynamicShadow);
1164
1165   Instruction *InsertPt = &*F.getEntryBlock().begin();
1166   IRBuilder<> EntryIRB(InsertPt);
1167   emitPrologue(EntryIRB,
1168                /*WithFrameRecord*/ ClRecordStackHistory &&
1169                    !AllocasToInstrument.empty());
1170
1171   bool Changed = false;
1172   if (!AllocasToInstrument.empty()) {
1173     Value *StackTag =
1174         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1175     Changed |= instrumentStack(AllocasToInstrument, AllocaDeclareMap, RetVec,
1176                                StackTag);
1177   }
1178
1179   // Pad and align each of the allocas that we instrumented to stop small
1180   // uninteresting allocas from hiding in instrumented alloca's padding and so
1181   // that we have enough space to store real tags for short granules.
1182   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1183   for (AllocaInst *AI : AllocasToInstrument) {
1184     uint64_t Size = getAllocaSizeInBytes(*AI);
1185     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1186     AI->setAlignment(
1187         MaybeAlign(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
1188     if (Size != AlignedSize) {
1189       Type *AllocatedType = AI->getAllocatedType();
1190       if (AI->isArrayAllocation()) {
1191         uint64_t ArraySize =
1192             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1193         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1194       }
1195       Type *TypeWithPadding = StructType::get(
1196           AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1197       auto *NewAI = new AllocaInst(
1198           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1199       NewAI->takeName(AI);
1200       NewAI->setAlignment(MaybeAlign(AI->getAlignment()));
1201       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1202       NewAI->setSwiftError(AI->isSwiftError());
1203       NewAI->copyMetadata(*AI);
1204       auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1205       AI->replaceAllUsesWith(Bitcast);
1206       AllocaToPaddedAllocaMap[AI] = NewAI;
1207     }
1208   }
1209
1210   if (!AllocaToPaddedAllocaMap.empty()) {
1211     for (auto &BB : F)
1212       for (auto &Inst : BB)
1213         if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1214           if (auto *AI =
1215                   dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation()))
1216             if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1217               DVI->setArgOperand(
1218                   0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI)));
1219     for (auto &P : AllocaToPaddedAllocaMap)
1220       P.first->eraseFromParent();
1221   }
1222
1223   // If we split the entry block, move any allocas that were originally in the
1224   // entry block back into the entry block so that they aren't treated as
1225   // dynamic allocas.
1226   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1227     InsertPt = &*F.getEntryBlock().begin();
1228     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1229               IE = EntryIRB.GetInsertBlock()->end();
1230          II != IE;) {
1231       Instruction *I = &*II++;
1232       if (auto *AI = dyn_cast<AllocaInst>(I))
1233         if (isa<ConstantInt>(AI->getArraySize()))
1234           I->moveBefore(InsertPt);
1235     }
1236   }
1237
1238   for (auto Inst : ToInstrument)
1239     Changed |= instrumentMemAccess(Inst);
1240
1241   LocalDynamicShadow = nullptr;
1242   StackBaseTag = nullptr;
1243
1244   return Changed;
1245 }
1246
1247 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1248   Constant *Initializer = GV->getInitializer();
1249   uint64_t SizeInBytes =
1250       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1251   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1252   if (SizeInBytes != NewSize) {
1253     // Pad the initializer out to the next multiple of 16 bytes and add the
1254     // required short granule tag.
1255     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1256     Init.back() = Tag;
1257     Constant *Padding = ConstantDataArray::get(*C, Init);
1258     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1259   }
1260
1261   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1262                                    GlobalValue::ExternalLinkage, Initializer,
1263                                    GV->getName() + ".hwasan");
1264   NewGV->copyAttributesFrom(GV);
1265   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1266   NewGV->copyMetadata(GV, 0);
1267   NewGV->setAlignment(
1268       std::max(GV->getAlignment(), Mapping.getObjectAlignment()));
1269
1270   // It is invalid to ICF two globals that have different tags. In the case
1271   // where the size of the global is a multiple of the tag granularity the
1272   // contents of the globals may be the same but the tags (i.e. symbol values)
1273   // may be different, and the symbols are not considered during ICF. In the
1274   // case where the size is not a multiple of the granularity, the short granule
1275   // tags would discriminate two globals with different tags, but there would
1276   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1277   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1278   // granule tag in the last byte.
1279   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1280
1281   // Descriptor format (assuming little-endian):
1282   // bytes 0-3: relative address of global
1283   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1284   // it isn't, we create multiple descriptors)
1285   // byte 7: tag
1286   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1287   const uint64_t MaxDescriptorSize = 0xfffff0;
1288   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1289        DescriptorPos += MaxDescriptorSize) {
1290     auto *Descriptor =
1291         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1292                            nullptr, GV->getName() + ".hwasan.descriptor");
1293     auto *GVRelPtr = ConstantExpr::getTrunc(
1294         ConstantExpr::getAdd(
1295             ConstantExpr::getSub(
1296                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1297                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1298             ConstantInt::get(Int64Ty, DescriptorPos)),
1299         Int32Ty);
1300     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1301     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1302     Descriptor->setComdat(NewGV->getComdat());
1303     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1304     Descriptor->setSection("hwasan_globals");
1305     Descriptor->setMetadata(LLVMContext::MD_associated,
1306                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1307     appendToCompilerUsed(M, Descriptor);
1308   }
1309
1310   Constant *Aliasee = ConstantExpr::getIntToPtr(
1311       ConstantExpr::getAdd(
1312           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1313           ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1314       GV->getType());
1315   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1316                                     GV->getLinkage(), "", Aliasee, &M);
1317   Alias->setVisibility(GV->getVisibility());
1318   Alias->takeName(GV);
1319   GV->replaceAllUsesWith(Alias);
1320   GV->eraseFromParent();
1321 }
1322
1323 void HWAddressSanitizer::instrumentGlobals() {
1324   // Start by creating a note that contains pointers to the list of global
1325   // descriptors. Adding a note to the output file will cause the linker to
1326   // create a PT_NOTE program header pointing to the note that we can use to
1327   // find the descriptor list starting from the program headers. A function
1328   // provided by the runtime initializes the shadow memory for the globals by
1329   // accessing the descriptor list via the note. The dynamic loader needs to
1330   // call this function whenever a library is loaded.
1331   //
1332   // The reason why we use a note for this instead of a more conventional
1333   // approach of having a global constructor pass a descriptor list pointer to
1334   // the runtime is because of an order of initialization problem. With
1335   // constructors we can encounter the following problematic scenario:
1336   //
1337   // 1) library A depends on library B and also interposes one of B's symbols
1338   // 2) B's constructors are called before A's (as required for correctness)
1339   // 3) during construction, B accesses one of its "own" globals (actually
1340   //    interposed by A) and triggers a HWASAN failure due to the initialization
1341   //    for A not having happened yet
1342   //
1343   // Even without interposition it is possible to run into similar situations in
1344   // cases where two libraries mutually depend on each other.
1345   //
1346   // We only need one note per binary, so put everything for the note in a
1347   // comdat.
1348   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanNoteName);
1349
1350   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
1351   auto Start =
1352       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
1353                          nullptr, "__start_hwasan_globals");
1354   Start->setVisibility(GlobalValue::HiddenVisibility);
1355   Start->setDSOLocal(true);
1356   auto Stop =
1357       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
1358                          nullptr, "__stop_hwasan_globals");
1359   Stop->setVisibility(GlobalValue::HiddenVisibility);
1360   Stop->setDSOLocal(true);
1361
1362   // Null-terminated so actually 8 bytes, which are required in order to align
1363   // the note properly.
1364   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
1365
1366   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
1367                                  Int32Ty, Int32Ty);
1368   auto *Note =
1369       new GlobalVariable(M, NoteTy, /*isConstantGlobal=*/true,
1370                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
1371   Note->setSection(".note.hwasan.globals");
1372   Note->setComdat(NoteComdat);
1373   Note->setAlignment(4);
1374   Note->setDSOLocal(true);
1375
1376   // The pointers in the note need to be relative so that the note ends up being
1377   // placed in rodata, which is the standard location for notes.
1378   auto CreateRelPtr = [&](Constant *Ptr) {
1379     return ConstantExpr::getTrunc(
1380         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
1381                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
1382         Int32Ty);
1383   };
1384   Note->setInitializer(ConstantStruct::getAnon(
1385       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
1386        ConstantInt::get(Int32Ty, 8),                           // n_descsz
1387        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
1388        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
1389   appendToCompilerUsed(M, Note);
1390
1391   // Create a zero-length global in hwasan_globals so that the linker will
1392   // always create start and stop symbols.
1393   auto Dummy = new GlobalVariable(
1394       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
1395       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
1396   Dummy->setSection("hwasan_globals");
1397   Dummy->setComdat(NoteComdat);
1398   Dummy->setMetadata(LLVMContext::MD_associated,
1399                      MDNode::get(*C, ValueAsMetadata::get(Note)));
1400   appendToCompilerUsed(M, Dummy);
1401
1402   std::vector<GlobalVariable *> Globals;
1403   for (GlobalVariable &GV : M.globals()) {
1404     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1405         GV.isThreadLocal())
1406       continue;
1407
1408     // Common symbols can't have aliases point to them, so they can't be tagged.
1409     if (GV.hasCommonLinkage())
1410       continue;
1411
1412     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1413     // which would be broken both by adding tags and potentially by the extra
1414     // padding/alignment that we insert.
1415     if (GV.hasSection())
1416       continue;
1417
1418     Globals.push_back(&GV);
1419   }
1420
1421   MD5 Hasher;
1422   Hasher.update(M.getSourceFileName());
1423   MD5::MD5Result Hash;
1424   Hasher.final(Hash);
1425   uint8_t Tag = Hash[0];
1426
1427   for (GlobalVariable *GV : Globals) {
1428     // Skip tag 0 in order to avoid collisions with untagged memory.
1429     if (Tag == 0)
1430       Tag = 1;
1431     instrumentGlobal(GV, Tag++);
1432   }
1433 }
1434
1435 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1436   // We need to untag stack frames as we unwind past them. That is the job of
1437   // the personality function wrapper, which either wraps an existing
1438   // personality function or acts as a personality function on its own. Each
1439   // function that has a personality function or that can be unwound past has
1440   // its personality function changed to a thunk that calls the personality
1441   // function wrapper in the runtime.
1442   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1443   for (Function &F : M) {
1444     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1445       continue;
1446
1447     if (F.hasPersonalityFn()) {
1448       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1449     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1450       PersonalityFns[nullptr].push_back(&F);
1451     }
1452   }
1453
1454   if (PersonalityFns.empty())
1455     return;
1456
1457   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1458       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1459       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1460   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1461   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1462
1463   for (auto &P : PersonalityFns) {
1464     std::string ThunkName = kHwasanPersonalityThunkName;
1465     if (P.first)
1466       ThunkName += ("." + P.first->getName()).str();
1467     FunctionType *ThunkFnTy = FunctionType::get(
1468         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1469     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1470                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1471     auto *ThunkFn = Function::Create(ThunkFnTy,
1472                                      IsLocal ? GlobalValue::InternalLinkage
1473                                              : GlobalValue::LinkOnceODRLinkage,
1474                                      ThunkName, &M);
1475     if (!IsLocal) {
1476       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1477       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1478     }
1479
1480     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1481     IRBuilder<> IRB(BB);
1482     CallInst *WrapperCall = IRB.CreateCall(
1483         HwasanPersonalityWrapper,
1484         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1485          ThunkFn->getArg(3), ThunkFn->getArg(4),
1486          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1487                  : Constant::getNullValue(Int8PtrTy),
1488          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1489          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1490     WrapperCall->setTailCall();
1491     IRB.CreateRet(WrapperCall);
1492
1493     for (Function *F : P.second)
1494       F->setPersonalityFn(ThunkFn);
1495   }
1496 }
1497
1498 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1499   Scale = kDefaultShadowScale;
1500   if (ClMappingOffset.getNumOccurrences() > 0) {
1501     InGlobal = false;
1502     InTls = false;
1503     Offset = ClMappingOffset;
1504   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1505     InGlobal = false;
1506     InTls = false;
1507     Offset = 0;
1508   } else if (ClWithIfunc) {
1509     InGlobal = true;
1510     InTls = false;
1511     Offset = kDynamicShadowSentinel;
1512   } else if (ClWithTls) {
1513     InGlobal = false;
1514     InTls = true;
1515     Offset = kDynamicShadowSentinel;
1516   } else {
1517     InGlobal = false;
1518     InTls = false;
1519     Offset = kDynamicShadowSentinel;
1520   }
1521 }