lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

   1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This file is a part of HWAddressSanitizer, an address sanity checker
  11 /// based on tagged addressing.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  15 #include "llvm/ADT/MapVector.h"
  16 #include "llvm/ADT/SmallVector.h"
  17 #include "llvm/ADT/StringExtras.h"
  18 #include "llvm/ADT/StringRef.h"
  19 #include "llvm/ADT/Triple.h"
  20 #include "llvm/BinaryFormat/ELF.h"
  21 #include "llvm/IR/Attributes.h"
  22 #include "llvm/IR/BasicBlock.h"
  23 #include "llvm/IR/Constant.h"
  24 #include "llvm/IR/Constants.h"
  25 #include "llvm/IR/DataLayout.h"
  26 #include "llvm/IR/DebugInfoMetadata.h"
  27 #include "llvm/IR/DerivedTypes.h"
  28 #include "llvm/IR/Function.h"
  29 #include "llvm/IR/IRBuilder.h"
  30 #include "llvm/IR/InlineAsm.h"
  31 #include "llvm/IR/InstVisitor.h"
  32 #include "llvm/IR/Instruction.h"
  33 #include "llvm/IR/Instructions.h"
  34 #include "llvm/IR/IntrinsicInst.h"
  35 #include "llvm/IR/Intrinsics.h"
  36 #include "llvm/IR/LLVMContext.h"
  37 #include "llvm/IR/MDBuilder.h"
  38 #include "llvm/IR/Module.h"
  39 #include "llvm/IR/Type.h"
  40 #include "llvm/IR/Value.h"
  41 #include "llvm/Pass.h"
  42 #include "llvm/Support/Casting.h"
  43 #include "llvm/Support/CommandLine.h"
  44 #include "llvm/Support/Debug.h"
  45 #include "llvm/Support/raw_ostream.h"
  46 #include "llvm/Transforms/Instrumentation.h"
  47 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  48 #include "llvm/Transforms/Utils/ModuleUtils.h"
  49 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
  50 #include <sstream>
  51
  52 using namespace llvm;
  53
  54 #define DEBUG_TYPE "hwasan"
  55
  56 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
  57 static const char *const kHwasanNoteName = "hwasan.note";
  58 static const char *const kHwasanInitName = "__hwasan_init";
  59 static const char *const kHwasanPersonalityThunkName =
  60     "__hwasan_personality_thunk";
  61
  62 static const char *const kHwasanShadowMemoryDynamicAddress =
  63     "__hwasan_shadow_memory_dynamic_address";
  64
  65 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
  66 static const size_t kNumberOfAccessSizes = 5;
  67
  68 static const size_t kDefaultShadowScale = 4;
  69 static const uint64_t kDynamicShadowSentinel =
  70     std::numeric_limits<uint64_t>::max();
  71 static const unsigned kPointerTagShift = 56;
  72
  73 static const unsigned kShadowBaseAlignment = 32;
  74
  75 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
  76     "hwasan-memory-access-callback-prefix",
  77     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
  78     cl::init("__hwasan_"));
  79
  80 static cl::opt<bool>
  81     ClInstrumentWithCalls("hwasan-instrument-with-calls",
  82                 cl::desc("instrument reads and writes with callbacks"),
  83                 cl::Hidden, cl::init(false));
  84
  85 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
  86                                        cl::desc("instrument read instructions"),
  87                                        cl::Hidden, cl::init(true));
  88
  89 static cl::opt<bool> ClInstrumentWrites(
  90     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
  91     cl::Hidden, cl::init(true));
  92
  93 static cl::opt<bool> ClInstrumentAtomics(
  94     "hwasan-instrument-atomics",
  95     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
  96     cl::init(true));
  97
  98 static cl::opt<bool> ClRecover(
  99     "hwasan-recover",
 100     cl::desc("Enable recovery mode (continue-after-error)."),
 101     cl::Hidden, cl::init(false));
 102
 103 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
 104                                        cl::desc("instrument stack (allocas)"),
 105                                        cl::Hidden, cl::init(true));
 106
 107 static cl::opt<bool> ClUARRetagToZero(
 108     "hwasan-uar-retag-to-zero",
 109     cl::desc("Clear alloca tags before returning from the function to allow "
 110              "non-instrumented and instrumented function calls mix. When set "
 111              "to false, allocas are retagged before returning from the "
 112              "function to detect use after return."),
 113     cl::Hidden, cl::init(true));
 114
 115 static cl::opt<bool> ClGenerateTagsWithCalls(
 116     "hwasan-generate-tags-with-calls",
 117     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
 118     cl::init(false));
 119
 120 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
 121                                cl::Hidden, cl::init(false));
 122
 123 static cl::opt<int> ClMatchAllTag(
 124     "hwasan-match-all-tag",
 125     cl::desc("don't report bad accesses via pointers with this tag"),
 126     cl::Hidden, cl::init(-1));
 127
 128 static cl::opt<bool> ClEnableKhwasan(
 129     "hwasan-kernel",
 130     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
 131     cl::Hidden, cl::init(false));
 132
 133 // These flags allow to change the shadow mapping and control how shadow memory
 134 // is accessed. The shadow mapping looks like:
 135 //    Shadow = (Mem >> scale) + offset
 136
 137 static cl::opt<uint64_t>
 138     ClMappingOffset("hwasan-mapping-offset",
 139                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
 140                     cl::Hidden, cl::init(0));
 141
 142 static cl::opt<bool>
 143     ClWithIfunc("hwasan-with-ifunc",
 144                 cl::desc("Access dynamic shadow through an ifunc global on "
 145                          "platforms that support this"),
 146                 cl::Hidden, cl::init(false));
 147
 148 static cl::opt<bool> ClWithTls(
 149     "hwasan-with-tls",
 150     cl::desc("Access dynamic shadow through an thread-local pointer on "
 151              "platforms that support this"),
 152     cl::Hidden, cl::init(true));
 153
 154 static cl::opt<bool>
 155     ClRecordStackHistory("hwasan-record-stack-history",
 156                          cl::desc("Record stack frames with tagged allocations "
 157                                   "in a thread-local ring buffer"),
 158                          cl::Hidden, cl::init(true));
 159 static cl::opt<bool>
 160     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
 161                               cl::desc("instrument memory intrinsics"),
 162                               cl::Hidden, cl::init(true));
 163
 164 static cl::opt<bool>
 165     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
 166                             cl::desc("instrument landing pads"), cl::Hidden,
 167                             cl::init(false), cl::ZeroOrMore);
 168
 169 static cl::opt<bool> ClInstrumentPersonalityFunctions(
 170     "hwasan-instrument-personality-functions",
 171     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
 172     cl::ZeroOrMore);
 173
 174 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
 175                                        cl::desc("inline all checks"),
 176                                        cl::Hidden, cl::init(false));
 177
 178 namespace {
 179
 180 /// An instrumentation pass implementing detection of addressability bugs
 181 /// using tagged pointers.
 182 class HWAddressSanitizer {
 183 public:
 184   explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
 185                               bool Recover = false) : M(M) {
 186     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
 187     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
 188         ClEnableKhwasan : CompileKernel;
 189
 190     initializeModule();
 191   }
 192
 193   bool sanitizeFunction(Function &F);
 194   void initializeModule();
 195
 196   void initializeCallbacks(Module &M);
 197
 198   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
 199   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 200
 201   void untagPointerOperand(Instruction *I, Value *Addr);
 202   Value *shadowBase();
 203   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
 204   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 205                                  unsigned AccessSizeIndex,
 206                                  Instruction *InsertBefore);
 207   void instrumentMemIntrinsic(MemIntrinsic *MI);
 208   bool instrumentMemAccess(Instruction *I);
 209   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
 210                                    uint64_t *TypeSize, unsigned *Alignment,
 211                                    Value **MaybeMask);
 212
 213   bool isInterestingAlloca(const AllocaInst &AI);
 214   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
 215   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
 216   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
 217   bool instrumentStack(
 218       SmallVectorImpl<AllocaInst *> &Allocas,
 219       DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap,
 220       SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
 221   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
 222   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
 223   Value *getNextTagWithCall(IRBuilder<> &IRB);
 224   Value *getStackBaseTag(IRBuilder<> &IRB);
 225   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
 226                      unsigned AllocaNo);
 227   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 228
 229   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
 230   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
 231
 232   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
 233   void instrumentGlobals();
 234
 235   void instrumentPersonalityFunctions();
 236
 237 private:
 238   LLVMContext *C;
 239   Module &M;
 240   Triple TargetTriple;
 241   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
 242   FunctionCallee HWAsanHandleVfork;
 243
 244   /// This struct defines the shadow mapping using the rule:
 245   ///   shadow = (mem >> Scale) + Offset.
 246   /// If InGlobal is true, then
 247   ///   extern char __hwasan_shadow[];
 248   ///   shadow = (mem >> Scale) + &__hwasan_shadow
 249   /// If InTls is true, then
 250   ///   extern char *__hwasan_tls;
 251   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
 252   struct ShadowMapping {
 253     int Scale;
 254     uint64_t Offset;
 255     bool InGlobal;
 256     bool InTls;
 257
 258     void init(Triple &TargetTriple);
 259     unsigned getObjectAlignment() const { return 1U << Scale; }
 260   };
 261   ShadowMapping Mapping;
 262
 263   Type *VoidTy = Type::getVoidTy(M.getContext());
 264   Type *IntptrTy;
 265   Type *Int8PtrTy;
 266   Type *Int8Ty;
 267   Type *Int32Ty;
 268   Type *Int64Ty = Type::getInt64Ty(M.getContext());
 269
 270   bool CompileKernel;
 271   bool Recover;
 272   bool InstrumentLandingPads;
 273
 274   Function *HwasanCtorFunction;
 275
 276   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
 277   FunctionCallee HwasanMemoryAccessCallbackSized[2];
 278
 279   FunctionCallee HwasanTagMemoryFunc;
 280   FunctionCallee HwasanGenerateTagFunc;
 281   FunctionCallee HwasanThreadEnterFunc;
 282
 283   Constant *ShadowGlobal;
 284
 285   Value *LocalDynamicShadow = nullptr;
 286   Value *StackBaseTag = nullptr;
 287   GlobalValue *ThreadPtrGlobal = nullptr;
 288 };
 289
 290 class HWAddressSanitizerLegacyPass : public FunctionPass {
 291 public:
 292   // Pass identification, replacement for typeid.
 293   static char ID;
 294
 295   explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
 296                                         bool Recover = false)
 297       : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {}
 298
 299   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 300
 301   bool doInitialization(Module &M) override {
 302     HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
 303     return true;
 304   }
 305
 306   bool runOnFunction(Function &F) override {
 307     return HWASan->sanitizeFunction(F);
 308   }
 309
 310   bool doFinalization(Module &M) override {
 311     HWASan.reset();
 312     return false;
 313   }
 314
 315 private:
 316   std::unique_ptr<HWAddressSanitizer> HWASan;
 317   bool CompileKernel;
 318   bool Recover;
 319 };
 320
 321 } // end anonymous namespace
 322
 323 char HWAddressSanitizerLegacyPass::ID = 0;
 324
 325 INITIALIZE_PASS_BEGIN(
 326     HWAddressSanitizerLegacyPass, "hwasan",
 327     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 328     false)
 329 INITIALIZE_PASS_END(
 330     HWAddressSanitizerLegacyPass, "hwasan",
 331     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 332     false)
 333
 334 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
 335                                                            bool Recover) {
 336   assert(!CompileKernel || Recover);
 337   return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
 338 }
 339
 340 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
 341     : CompileKernel(CompileKernel), Recover(Recover) {}
 342
 343 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
 344                                               ModuleAnalysisManager &MAM) {
 345   HWAddressSanitizer HWASan(M, CompileKernel, Recover);
 346   bool Modified = false;
 347   for (Function &F : M)
 348     Modified |= HWASan.sanitizeFunction(F);
 349   if (Modified)
 350     return PreservedAnalyses::none();
 351   return PreservedAnalyses::all();
 352 }
 353
 354 /// Module-level initialization.
 355 ///
 356 /// inserts a call to __hwasan_init to the module's constructor list.
 357 void HWAddressSanitizer::initializeModule() {
 358   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
 359   auto &DL = M.getDataLayout();
 360
 361   TargetTriple = Triple(M.getTargetTriple());
 362
 363   Mapping.init(TargetTriple);
 364
 365   C = &(M.getContext());
 366   IRBuilder<> IRB(*C);
 367   IntptrTy = IRB.getIntPtrTy(DL);
 368   Int8PtrTy = IRB.getInt8PtrTy();
 369   Int8Ty = IRB.getInt8Ty();
 370   Int32Ty = IRB.getInt32Ty();
 371
 372   HwasanCtorFunction = nullptr;
 373
 374   // Older versions of Android do not have the required runtime support for
 375   // global or personality function instrumentation. On other platforms we
 376   // currently require using the latest version of the runtime.
 377   bool NewRuntime =
 378       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
 379
 380   // If we don't have personality function support, fall back to landing pads.
 381   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
 382                               ? ClInstrumentLandingPads
 383                               : !NewRuntime;
 384
 385   if (!CompileKernel) {
 386     std::tie(HwasanCtorFunction, std::ignore) =
 387         getOrCreateSanitizerCtorAndInitFunctions(
 388             M, kHwasanModuleCtorName, kHwasanInitName,
 389             /*InitArgTypes=*/{},
 390             /*InitArgs=*/{},
 391             // This callback is invoked when the functions are created the first
 392             // time. Hook them into the global ctors list in that case:
 393             [&](Function *Ctor, FunctionCallee) {
 394               Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
 395               Ctor->setComdat(CtorComdat);
 396               appendToGlobalCtors(M, Ctor, 0, Ctor);
 397             });
 398
 399     bool InstrumentGlobals =
 400         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
 401     if (InstrumentGlobals)
 402       instrumentGlobals();
 403
 404     bool InstrumentPersonalityFunctions =
 405         ClInstrumentPersonalityFunctions.getNumOccurrences()
 406             ? ClInstrumentPersonalityFunctions
 407             : NewRuntime;
 408     if (InstrumentPersonalityFunctions)
 409       instrumentPersonalityFunctions();
 410   }
 411
 412   if (!TargetTriple.isAndroid()) {
 413     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
 414       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
 415                                     GlobalValue::ExternalLinkage, nullptr,
 416                                     "__hwasan_tls", nullptr,
 417                                     GlobalVariable::InitialExecTLSModel);
 418       appendToCompilerUsed(M, GV);
 419       return GV;
 420     });
 421     ThreadPtrGlobal = cast<GlobalVariable>(C);
 422   }
 423 }
 424
 425 void HWAddressSanitizer::initializeCallbacks(Module &M) {
 426   IRBuilder<> IRB(*C);
 427   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
 428     const std::string TypeStr = AccessIsWrite ? "store" : "load";
 429     const std::string EndingStr = Recover ? "_noabort" : "";
 430
 431     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
 432         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
 433         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
 434
 435     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
 436          AccessSizeIndex++) {
 437       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
 438           M.getOrInsertFunction(
 439               ClMemoryAccessCallbackPrefix + TypeStr +
 440                   itostr(1ULL << AccessSizeIndex) + EndingStr,
 441               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
 442     }
 443   }
 444
 445   HwasanTagMemoryFunc = M.getOrInsertFunction(
 446       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
 447   HwasanGenerateTagFunc =
 448       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
 449
 450   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
 451                                      ArrayType::get(IRB.getInt8Ty(), 0));
 452
 453   const std::string MemIntrinCallbackPrefix =
 454       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
 455   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
 456                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 457                                         IRB.getInt8PtrTy(), IntptrTy);
 458   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
 459                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 460                                        IRB.getInt8PtrTy(), IntptrTy);
 461   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
 462                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 463                                        IRB.getInt32Ty(), IntptrTy);
 464
 465   HWAsanHandleVfork =
 466       M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
 467
 468   HwasanThreadEnterFunc =
 469       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy());
 470 }
 471
 472 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
 473   // An empty inline asm with input reg == output reg.
 474   // An opaque no-op cast, basically.
 475   InlineAsm *Asm = InlineAsm::get(
 476       FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
 477       StringRef(""), StringRef("=r,0"),
 478       /*hasSideEffects=*/false);
 479   return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
 480 }
 481
 482 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
 483   // Generate code only when dynamic addressing is needed.
 484   if (Mapping.Offset != kDynamicShadowSentinel)
 485     return nullptr;
 486
 487   if (Mapping.InGlobal) {
 488     return getDynamicShadowIfunc(IRB);
 489   } else {
 490     Value *GlobalDynamicAddress =
 491         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
 492             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
 493     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
 494   }
 495 }
 496
 497 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
 498                                                      bool *IsWrite,
 499                                                      uint64_t *TypeSize,
 500                                                      unsigned *Alignment,
 501                                                      Value **MaybeMask) {
 502   // Skip memory accesses inserted by another instrumentation.
 503   if (I->getMetadata("nosanitize")) return nullptr;
 504
 505   // Do not instrument the load fetching the dynamic shadow address.
 506   if (LocalDynamicShadow == I)
 507     return nullptr;
 508
 509   Value *PtrOperand = nullptr;
 510   const DataLayout &DL = I->getModule()->getDataLayout();
 511   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 512     if (!ClInstrumentReads) return nullptr;
 513     *IsWrite = false;
 514     *TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
 515     *Alignment = LI->getAlignment();
 516     PtrOperand = LI->getPointerOperand();
 517   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 518     if (!ClInstrumentWrites) return nullptr;
 519     *IsWrite = true;
 520     *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
 521     *Alignment = SI->getAlignment();
 522     PtrOperand = SI->getPointerOperand();
 523   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
 524     if (!ClInstrumentAtomics) return nullptr;
 525     *IsWrite = true;
 526     *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
 527     *Alignment = 0;
 528     PtrOperand = RMW->getPointerOperand();
 529   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
 530     if (!ClInstrumentAtomics) return nullptr;
 531     *IsWrite = true;
 532     *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
 533     *Alignment = 0;
 534     PtrOperand = XCHG->getPointerOperand();
 535   }
 536
 537   if (PtrOperand) {
 538     // Do not instrument accesses from different address spaces; we cannot deal
 539     // with them.
 540     Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
 541     if (PtrTy->getPointerAddressSpace() != 0)
 542       return nullptr;
 543
 544     // Ignore swifterror addresses.
 545     // swifterror memory addresses are mem2reg promoted by instruction
 546     // selection. As such they cannot have regular uses like an instrumentation
 547     // function and it makes no sense to track them as memory.
 548     if (PtrOperand->isSwiftError())
 549       return nullptr;
 550   }
 551
 552   return PtrOperand;
 553 }
 554
 555 static unsigned getPointerOperandIndex(Instruction *I) {
 556   if (LoadInst *LI = dyn_cast<LoadInst>(I))
 557     return LI->getPointerOperandIndex();
 558   if (StoreInst *SI = dyn_cast<StoreInst>(I))
 559     return SI->getPointerOperandIndex();
 560   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
 561     return RMW->getPointerOperandIndex();
 562   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
 563     return XCHG->getPointerOperandIndex();
 564   report_fatal_error("Unexpected instruction");
 565   return -1;
 566 }
 567
 568 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
 569   size_t Res = countTrailingZeros(TypeSize / 8);
 570   assert(Res < kNumberOfAccessSizes);
 571   return Res;
 572 }
 573
 574 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
 575   if (TargetTriple.isAArch64())
 576     return;
 577
 578   IRBuilder<> IRB(I);
 579   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
 580   Value *UntaggedPtr =
 581       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
 582   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 583 }
 584
 585 Value *HWAddressSanitizer::shadowBase() {
 586   if (LocalDynamicShadow)
 587     return LocalDynamicShadow;
 588   return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
 589                                    Int8PtrTy);
 590 }
 591
 592 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
 593   // Mem >> Scale
 594   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
 595   if (Mapping.Offset == 0)
 596     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
 597   // (Mem >> Scale) + Offset
 598   return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
 599 }
 600
 601 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 602                                                    unsigned AccessSizeIndex,
 603                                                    Instruction *InsertBefore) {
 604   const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
 605   IRBuilder<> IRB(InsertBefore);
 606
 607   if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
 608       TargetTriple.isOSBinFormatELF() && !Recover) {
 609     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 610     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
 611     IRB.CreateCall(
 612         Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess),
 613         {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
 614     return;
 615   }
 616
 617   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
 618   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
 619                                   IRB.getInt8Ty());
 620   Value *AddrLong = untagPointer(IRB, PtrLong);
 621   Value *Shadow = memToShadow(AddrLong, IRB);
 622   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
 623   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 624
 625   int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
 626       ClMatchAllTag : (CompileKernel ? 0xFF : -1);
 627   if (matchAllTag != -1) {
 628     Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
 629         ConstantInt::get(PtrTag->getType(), matchAllTag));
 630     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
 631   }
 632
 633   Instruction *CheckTerm =
 634       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
 635                                 MDBuilder(*C).createBranchWeights(1, 100000));
 636
 637   IRB.SetInsertPoint(CheckTerm);
 638   Value *OutOfShortGranuleTagRange =
 639       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
 640   Instruction *CheckFailTerm =
 641       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
 642                                 MDBuilder(*C).createBranchWeights(1, 100000));
 643
 644   IRB.SetInsertPoint(CheckTerm);
 645   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
 646   PtrLowBits = IRB.CreateAdd(
 647       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
 648   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
 649   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
 650                             MDBuilder(*C).createBranchWeights(1, 100000),
 651                             nullptr, nullptr, CheckFailTerm->getParent());
 652
 653   IRB.SetInsertPoint(CheckTerm);
 654   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
 655   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
 656   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
 657   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
 658   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
 659                             MDBuilder(*C).createBranchWeights(1, 100000),
 660                             nullptr, nullptr, CheckFailTerm->getParent());
 661
 662   IRB.SetInsertPoint(CheckFailTerm);
 663   InlineAsm *Asm;
 664   switch (TargetTriple.getArch()) {
 665     case Triple::x86_64:
 666       // The signal handler will find the data address in rdi.
 667       Asm = InlineAsm::get(
 668           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 669           "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
 670           "{rdi}",
 671           /*hasSideEffects=*/true);
 672       break;
 673     case Triple::aarch64:
 674     case Triple::aarch64_be:
 675       // The signal handler will find the data address in x0.
 676       Asm = InlineAsm::get(
 677           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 678           "brk #" + itostr(0x900 + AccessInfo),
 679           "{x0}",
 680           /*hasSideEffects=*/true);
 681       break;
 682     default:
 683       report_fatal_error("unsupported architecture");
 684   }
 685   IRB.CreateCall(Asm, PtrLong);
 686   if (Recover)
 687     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
 688 }
 689
 690 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
 691   IRBuilder<> IRB(MI);
 692   if (isa<MemTransferInst>(MI)) {
 693     IRB.CreateCall(
 694         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
 695         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 696          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
 697          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 698   } else if (isa<MemSetInst>(MI)) {
 699     IRB.CreateCall(
 700         HWAsanMemset,
 701         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 702          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
 703          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 704   }
 705   MI->eraseFromParent();
 706 }
 707
 708 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
 709   LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
 710   bool IsWrite = false;
 711   unsigned Alignment = 0;
 712   uint64_t TypeSize = 0;
 713   Value *MaybeMask = nullptr;
 714
 715   if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) {
 716     instrumentMemIntrinsic(cast<MemIntrinsic>(I));
 717     return true;
 718   }
 719
 720   Value *Addr =
 721       isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
 722
 723   if (!Addr)
 724     return false;
 725
 726   if (MaybeMask)
 727     return false; //FIXME
 728
 729   IRBuilder<> IRB(I);
 730   if (isPowerOf2_64(TypeSize) &&
 731       (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
 732       (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
 733        Alignment >= TypeSize / 8)) {
 734     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
 735     if (ClInstrumentWithCalls) {
 736       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
 737                      IRB.CreatePointerCast(Addr, IntptrTy));
 738     } else {
 739       instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I);
 740     }
 741   } else {
 742     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
 743                    {IRB.CreatePointerCast(Addr, IntptrTy),
 744                     ConstantInt::get(IntptrTy, TypeSize / 8)});
 745   }
 746   untagPointerOperand(I, Addr);
 747
 748   return true;
 749 }
 750
 751 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
 752   uint64_t ArraySize = 1;
 753   if (AI.isArrayAllocation()) {
 754     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
 755     assert(CI && "non-constant array size");
 756     ArraySize = CI->getZExtValue();
 757   }
 758   Type *Ty = AI.getAllocatedType();
 759   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
 760   return SizeInBytes * ArraySize;
 761 }
 762
 763 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
 764                                    Value *Tag, size_t Size) {
 765   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
 766
 767   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
 768   if (ClInstrumentWithCalls) {
 769     IRB.CreateCall(HwasanTagMemoryFunc,
 770                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
 771                     ConstantInt::get(IntptrTy, AlignedSize)});
 772   } else {
 773     size_t ShadowSize = Size >> Mapping.Scale;
 774     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
 775     // If this memset is not inlined, it will be intercepted in the hwasan
 776     // runtime library. That's OK, because the interceptor skips the checks if
 777     // the address is in the shadow region.
 778     // FIXME: the interceptor is not as fast as real memset. Consider lowering
 779     // llvm.memset right here into either a sequence of stores, or a call to
 780     // hwasan_tag_memory.
 781     if (ShadowSize)
 782       IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
 783     if (Size != AlignedSize) {
 784       IRB.CreateStore(
 785           ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
 786           IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
 787       IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
 788                                    Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
 789                                    AlignedSize - 1));
 790     }
 791   }
 792   return true;
 793 }
 794
 795 static unsigned RetagMask(unsigned AllocaNo) {
 796   // A list of 8-bit numbers that have at most one run of non-zero bits.
 797   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
 798   // masks.
 799   // The list does not include the value 255, which is used for UAR.
 800   //
 801   // Because we are more likely to use earlier elements of this list than later
 802   // ones, it is sorted in increasing order of probability of collision with a
 803   // mask allocated (temporally) nearby. The program that generated this list
 804   // can be found at:
 805   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
 806   static unsigned FastMasks[] = {0,  128, 64,  192, 32,  96,  224, 112, 240,
 807                                  48, 16,  120, 248, 56,  24,  8,   124, 252,
 808                                  60, 28,  12,  4,   126, 254, 62,  30,  14,
 809                                  6,  2,   127, 63,  31,  15,  7,   3,   1};
 810   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
 811 }
 812
 813 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
 814   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
 815 }
 816
 817 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
 818   if (ClGenerateTagsWithCalls)
 819     return getNextTagWithCall(IRB);
 820   if (StackBaseTag)
 821     return StackBaseTag;
 822   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
 823   // first).
 824   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 825   auto GetStackPointerFn = Intrinsic::getDeclaration(
 826       M, Intrinsic::frameaddress,
 827       IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
 828   Value *StackPointer = IRB.CreateCall(
 829       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
 830
 831   // Extract some entropy from the stack pointer for the tags.
 832   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
 833   // between functions).
 834   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
 835   Value *StackTag =
 836       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
 837                     "hwasan.stack.base.tag");
 838   return StackTag;
 839 }
 840
 841 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
 842                                         AllocaInst *AI, unsigned AllocaNo) {
 843   if (ClGenerateTagsWithCalls)
 844     return getNextTagWithCall(IRB);
 845   return IRB.CreateXor(StackTag,
 846                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
 847 }
 848
 849 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
 850   if (ClUARRetagToZero)
 851     return ConstantInt::get(IntptrTy, 0);
 852   if (ClGenerateTagsWithCalls)
 853     return getNextTagWithCall(IRB);
 854   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
 855 }
 856
 857 // Add a tag to an address.
 858 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
 859                                       Value *PtrLong, Value *Tag) {
 860   Value *TaggedPtrLong;
 861   if (CompileKernel) {
 862     // Kernel addresses have 0xFF in the most significant byte.
 863     Value *ShiftedTag = IRB.CreateOr(
 864         IRB.CreateShl(Tag, kPointerTagShift),
 865         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
 866     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
 867   } else {
 868     // Userspace can simply do OR (tag << 56);
 869     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
 870     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
 871   }
 872   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
 873 }
 874
 875 // Remove tag from an address.
 876 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
 877   Value *UntaggedPtrLong;
 878   if (CompileKernel) {
 879     // Kernel addresses have 0xFF in the most significant byte.
 880     UntaggedPtrLong = IRB.CreateOr(PtrLong,
 881         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
 882   } else {
 883     // Userspace addresses have 0x00.
 884     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
 885         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
 886   }
 887   return UntaggedPtrLong;
 888 }
 889
 890 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
 891   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 892   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
 893     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
 894     // in Bionic's libc/private/bionic_tls.h.
 895     Function *ThreadPointerFunc =
 896         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
 897     Value *SlotPtr = IRB.CreatePointerCast(
 898         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
 899                                IRB.CreateCall(ThreadPointerFunc), 0x30),
 900         Ty->getPointerTo(0));
 901     return SlotPtr;
 902   }
 903   if (ThreadPtrGlobal)
 904     return ThreadPtrGlobal;
 905
 906
 907   return nullptr;
 908 }
 909
 910 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
 911   if (!Mapping.InTls) {
 912     LocalDynamicShadow = getDynamicShadowNonTls(IRB);
 913     return;
 914   }
 915
 916   if (!WithFrameRecord && TargetTriple.isAndroid()) {
 917     LocalDynamicShadow = getDynamicShadowIfunc(IRB);
 918     return;
 919   }
 920
 921   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
 922   assert(SlotPtr);
 923
 924   Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 925
 926   Function *F = IRB.GetInsertBlock()->getParent();
 927   if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
 928     Value *ThreadLongEqZero =
 929         IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
 930     auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
 931         ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
 932         false, MDBuilder(*C).createBranchWeights(1, 100000)));
 933
 934     IRB.SetInsertPoint(Br);
 935     // FIXME: This should call a new runtime function with a custom calling
 936     // convention to avoid needing to spill all arguments here.
 937     IRB.CreateCall(HwasanThreadEnterFunc);
 938     LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 939
 940     IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin());
 941     PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2);
 942     ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent());
 943     ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent());
 944     ThreadLong = ThreadLongPhi;
 945   }
 946
 947   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
 948   Value *ThreadLongMaybeUntagged =
 949       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
 950
 951   if (WithFrameRecord) {
 952     StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
 953
 954     // Prepare ring buffer data.
 955     Value *PC;
 956     if (TargetTriple.getArch() == Triple::aarch64)
 957       PC = readRegister(IRB, "pc");
 958     else
 959       PC = IRB.CreatePtrToInt(F, IntptrTy);
 960     Module *M = F->getParent();
 961     auto GetStackPointerFn = Intrinsic::getDeclaration(
 962         M, Intrinsic::frameaddress,
 963         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
 964     Value *SP = IRB.CreatePtrToInt(
 965         IRB.CreateCall(GetStackPointerFn,
 966                        {Constant::getNullValue(IRB.getInt32Ty())}),
 967         IntptrTy);
 968     // Mix SP and PC.
 969     // Assumptions:
 970     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
 971     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
 972     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
 973     //       0xSSSSPPPPPPPPPPPP
 974     SP = IRB.CreateShl(SP, 44);
 975
 976     // Store data to ring buffer.
 977     Value *RecordPtr =
 978         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
 979     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
 980
 981     // Update the ring buffer. Top byte of ThreadLong defines the size of the
 982     // buffer in pages, it must be a power of two, and the start of the buffer
 983     // must be aligned by twice that much. Therefore wrap around of the ring
 984     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
 985     // The use of AShr instead of LShr is due to
 986     //   https://bugs.llvm.org/show_bug.cgi?id=39030
 987     // Runtime library makes sure not to use the highest bit.
 988     Value *WrapMask = IRB.CreateXor(
 989         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
 990         ConstantInt::get(IntptrTy, (uint64_t)-1));
 991     Value *ThreadLongNew = IRB.CreateAnd(
 992         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
 993     IRB.CreateStore(ThreadLongNew, SlotPtr);
 994   }
 995
 996   // Get shadow base address by aligning RecordPtr up.
 997   // Note: this is not correct if the pointer is already aligned.
 998   // Runtime library will make sure this never happens.
 999   LocalDynamicShadow = IRB.CreateAdd(
1000       IRB.CreateOr(
1001           ThreadLongMaybeUntagged,
1002           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1003       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1004   LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy);
1005 }
1006
1007 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1008   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1009   Function *ReadRegister =
1010       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1011   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1012   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1013   return IRB.CreateCall(ReadRegister, Args);
1014 }
1015
1016 bool HWAddressSanitizer::instrumentLandingPads(
1017     SmallVectorImpl<Instruction *> &LandingPadVec) {
1018   for (auto *LP : LandingPadVec) {
1019     IRBuilder<> IRB(LP->getNextNode());
1020     IRB.CreateCall(
1021         HWAsanHandleVfork,
1022         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1023                                                                       : "sp")});
1024   }
1025   return true;
1026 }
1027
1028 bool HWAddressSanitizer::instrumentStack(
1029     SmallVectorImpl<AllocaInst *> &Allocas,
1030     DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> &AllocaDeclareMap,
1031     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1032   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1033   // alloca addresses using that. Unfortunately, offsets are not known yet
1034   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1035   // temp, shift-OR it into each alloca address and xor with the retag mask.
1036   // This generates one extra instruction per alloca use.
1037   for (unsigned N = 0; N < Allocas.size(); ++N) {
1038     auto *AI = Allocas[N];
1039     IRBuilder<> IRB(AI->getNextNode());
1040
1041     // Replace uses of the alloca with tagged address.
1042     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1043     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1044     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1045     std::string Name =
1046         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1047     Replacement->setName(Name + ".hwasan");
1048
1049     AI->replaceUsesWithIf(Replacement,
1050                           [AILong](Use &U) { return U.getUser() != AILong; });
1051
1052     for (auto *DDI : AllocaDeclareMap.lookup(AI)) {
1053       DIExpression *OldExpr = DDI->getExpression();
1054       DIExpression *NewExpr = DIExpression::append(
1055           OldExpr, {dwarf::DW_OP_LLVM_tag_offset, RetagMask(N)});
1056       DDI->setArgOperand(2, MetadataAsValue::get(*C, NewExpr));
1057     }
1058
1059     size_t Size = getAllocaSizeInBytes(*AI);
1060     tagAlloca(IRB, AI, Tag, Size);
1061
1062     for (auto RI : RetVec) {
1063       IRB.SetInsertPoint(RI);
1064
1065       // Re-tag alloca memory with the special UAR tag.
1066       Value *Tag = getUARTag(IRB, StackTag);
1067       tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1068     }
1069   }
1070
1071   return true;
1072 }
1073
1074 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1075   return (AI.getAllocatedType()->isSized() &&
1076           // FIXME: instrument dynamic allocas, too
1077           AI.isStaticAlloca() &&
1078           // alloca() may be called with 0 size, ignore it.
1079           getAllocaSizeInBytes(AI) > 0 &&
1080           // We are only interested in allocas not promotable to registers.
1081           // Promotable allocas are common under -O0.
1082           !isAllocaPromotable(&AI) &&
1083           // inalloca allocas are not treated as static, and we don't want
1084           // dynamic alloca instrumentation for them as well.
1085           !AI.isUsedWithInAlloca() &&
1086           // swifterror allocas are register promoted by ISel
1087           !AI.isSwiftError());
1088 }
1089
1090 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1091   if (&F == HwasanCtorFunction)
1092     return false;
1093
1094   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1095     return false;
1096
1097   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1098
1099   SmallVector<Instruction*, 16> ToInstrument;
1100   SmallVector<AllocaInst*, 8> AllocasToInstrument;
1101   SmallVector<Instruction*, 8> RetVec;
1102   SmallVector<Instruction*, 8> LandingPadVec;
1103   DenseMap<AllocaInst *, std::vector<DbgDeclareInst *>> AllocaDeclareMap;
1104   for (auto &BB : F) {
1105     for (auto &Inst : BB) {
1106       if (ClInstrumentStack)
1107         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1108           if (isInterestingAlloca(*AI))
1109             AllocasToInstrument.push_back(AI);
1110           continue;
1111         }
1112
1113       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1114           isa<CleanupReturnInst>(Inst))
1115         RetVec.push_back(&Inst);
1116
1117       if (auto *DDI = dyn_cast<DbgDeclareInst>(&Inst))
1118         if (auto *Alloca = dyn_cast_or_null<AllocaInst>(DDI->getAddress()))
1119           AllocaDeclareMap[Alloca].push_back(DDI);
1120
1121       if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1122         LandingPadVec.push_back(&Inst);
1123
1124       Value *MaybeMask = nullptr;
1125       bool IsWrite;
1126       unsigned Alignment;
1127       uint64_t TypeSize;
1128       Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize,
1129                                               &Alignment, &MaybeMask);
1130       if (Addr || isa<MemIntrinsic>(Inst))
1131         ToInstrument.push_back(&Inst);
1132     }
1133   }
1134
1135   initializeCallbacks(*F.getParent());
1136
1137   if (!LandingPadVec.empty())
1138     instrumentLandingPads(LandingPadVec);
1139
1140   if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1141       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1142     // __hwasan_personality_thunk is a no-op for functions without an
1143     // instrumented stack, so we can drop it.
1144     F.setPersonalityFn(nullptr);
1145   }
1146
1147   if (AllocasToInstrument.empty() && ToInstrument.empty())
1148     return false;
1149
1150   assert(!LocalDynamicShadow);
1151
1152   Instruction *InsertPt = &*F.getEntryBlock().begin();
1153   IRBuilder<> EntryIRB(InsertPt);
1154   emitPrologue(EntryIRB,
1155                /*WithFrameRecord*/ ClRecordStackHistory &&
1156                    !AllocasToInstrument.empty());
1157
1158   bool Changed = false;
1159   if (!AllocasToInstrument.empty()) {
1160     Value *StackTag =
1161         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1162     Changed |= instrumentStack(AllocasToInstrument, AllocaDeclareMap, RetVec,
1163                                StackTag);
1164   }
1165
1166   // Pad and align each of the allocas that we instrumented to stop small
1167   // uninteresting allocas from hiding in instrumented alloca's padding and so
1168   // that we have enough space to store real tags for short granules.
1169   DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1170   for (AllocaInst *AI : AllocasToInstrument) {
1171     uint64_t Size = getAllocaSizeInBytes(*AI);
1172     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1173     AI->setAlignment(
1174         std::max(AI->getAlignment(), Mapping.getObjectAlignment()));
1175     if (Size != AlignedSize) {
1176       Type *AllocatedType = AI->getAllocatedType();
1177       if (AI->isArrayAllocation()) {
1178         uint64_t ArraySize =
1179             cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1180         AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1181       }
1182       Type *TypeWithPadding = StructType::get(
1183           AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1184       auto *NewAI = new AllocaInst(
1185           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1186       NewAI->takeName(AI);
1187       NewAI->setAlignment(AI->getAlignment());
1188       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1189       NewAI->setSwiftError(AI->isSwiftError());
1190       NewAI->copyMetadata(*AI);
1191       auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1192       AI->replaceAllUsesWith(Bitcast);
1193       AllocaToPaddedAllocaMap[AI] = NewAI;
1194     }
1195   }
1196
1197   if (!AllocaToPaddedAllocaMap.empty()) {
1198     for (auto &BB : F)
1199       for (auto &Inst : BB)
1200         if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1201           if (auto *AI =
1202                   dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation()))
1203             if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1204               DVI->setArgOperand(
1205                   0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI)));
1206     for (auto &P : AllocaToPaddedAllocaMap)
1207       P.first->eraseFromParent();
1208   }
1209
1210   // If we split the entry block, move any allocas that were originally in the
1211   // entry block back into the entry block so that they aren't treated as
1212   // dynamic allocas.
1213   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1214     InsertPt = &*F.getEntryBlock().begin();
1215     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1216               IE = EntryIRB.GetInsertBlock()->end();
1217          II != IE;) {
1218       Instruction *I = &*II++;
1219       if (auto *AI = dyn_cast<AllocaInst>(I))
1220         if (isa<ConstantInt>(AI->getArraySize()))
1221           I->moveBefore(InsertPt);
1222     }
1223   }
1224
1225   for (auto Inst : ToInstrument)
1226     Changed |= instrumentMemAccess(Inst);
1227
1228   LocalDynamicShadow = nullptr;
1229   StackBaseTag = nullptr;
1230
1231   return Changed;
1232 }
1233
1234 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1235   Constant *Initializer = GV->getInitializer();
1236   uint64_t SizeInBytes =
1237       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1238   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1239   if (SizeInBytes != NewSize) {
1240     // Pad the initializer out to the next multiple of 16 bytes and add the
1241     // required short granule tag.
1242     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1243     Init.back() = Tag;
1244     Constant *Padding = ConstantDataArray::get(*C, Init);
1245     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1246   }
1247
1248   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1249                                    GlobalValue::ExternalLinkage, Initializer,
1250                                    GV->getName() + ".hwasan");
1251   NewGV->copyAttributesFrom(GV);
1252   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1253   NewGV->copyMetadata(GV, 0);
1254   NewGV->setAlignment(
1255       std::max(GV->getAlignment(), Mapping.getObjectAlignment()));
1256
1257   // It is invalid to ICF two globals that have different tags. In the case
1258   // where the size of the global is a multiple of the tag granularity the
1259   // contents of the globals may be the same but the tags (i.e. symbol values)
1260   // may be different, and the symbols are not considered during ICF. In the
1261   // case where the size is not a multiple of the granularity, the short granule
1262   // tags would discriminate two globals with different tags, but there would
1263   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1264   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1265   // granule tag in the last byte.
1266   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1267
1268   // Descriptor format (assuming little-endian):
1269   // bytes 0-3: relative address of global
1270   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1271   // it isn't, we create multiple descriptors)
1272   // byte 7: tag
1273   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1274   const uint64_t MaxDescriptorSize = 0xfffff0;
1275   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1276        DescriptorPos += MaxDescriptorSize) {
1277     auto *Descriptor =
1278         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1279                            nullptr, GV->getName() + ".hwasan.descriptor");
1280     auto *GVRelPtr = ConstantExpr::getTrunc(
1281         ConstantExpr::getAdd(
1282             ConstantExpr::getSub(
1283                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1284                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1285             ConstantInt::get(Int64Ty, DescriptorPos)),
1286         Int32Ty);
1287     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1288     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1289     Descriptor->setComdat(NewGV->getComdat());
1290     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1291     Descriptor->setSection("hwasan_globals");
1292     Descriptor->setMetadata(LLVMContext::MD_associated,
1293                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1294     appendToCompilerUsed(M, Descriptor);
1295   }
1296
1297   Constant *Aliasee = ConstantExpr::getIntToPtr(
1298       ConstantExpr::getAdd(
1299           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1300           ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1301       GV->getType());
1302   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1303                                     GV->getLinkage(), "", Aliasee, &M);
1304   Alias->setVisibility(GV->getVisibility());
1305   Alias->takeName(GV);
1306   GV->replaceAllUsesWith(Alias);
1307   GV->eraseFromParent();
1308 }
1309
1310 void HWAddressSanitizer::instrumentGlobals() {
1311   // Start by creating a note that contains pointers to the list of global
1312   // descriptors. Adding a note to the output file will cause the linker to
1313   // create a PT_NOTE program header pointing to the note that we can use to
1314   // find the descriptor list starting from the program headers. A function
1315   // provided by the runtime initializes the shadow memory for the globals by
1316   // accessing the descriptor list via the note. The dynamic loader needs to
1317   // call this function whenever a library is loaded.
1318   //
1319   // The reason why we use a note for this instead of a more conventional
1320   // approach of having a global constructor pass a descriptor list pointer to
1321   // the runtime is because of an order of initialization problem. With
1322   // constructors we can encounter the following problematic scenario:
1323   //
1324   // 1) library A depends on library B and also interposes one of B's symbols
1325   // 2) B's constructors are called before A's (as required for correctness)
1326   // 3) during construction, B accesses one of its "own" globals (actually
1327   //    interposed by A) and triggers a HWASAN failure due to the initialization
1328   //    for A not having happened yet
1329   //
1330   // Even without interposition it is possible to run into similar situations in
1331   // cases where two libraries mutually depend on each other.
1332   //
1333   // We only need one note per binary, so put everything for the note in a
1334   // comdat.
1335   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanNoteName);
1336
1337   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
1338   auto Start =
1339       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
1340                          nullptr, "__start_hwasan_globals");
1341   Start->setVisibility(GlobalValue::HiddenVisibility);
1342   Start->setDSOLocal(true);
1343   auto Stop =
1344       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
1345                          nullptr, "__stop_hwasan_globals");
1346   Stop->setVisibility(GlobalValue::HiddenVisibility);
1347   Stop->setDSOLocal(true);
1348
1349   // Null-terminated so actually 8 bytes, which are required in order to align
1350   // the note properly.
1351   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
1352
1353   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
1354                                  Int32Ty, Int32Ty);
1355   auto *Note =
1356       new GlobalVariable(M, NoteTy, /*isConstantGlobal=*/true,
1357                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
1358   Note->setSection(".note.hwasan.globals");
1359   Note->setComdat(NoteComdat);
1360   Note->setAlignment(4);
1361   Note->setDSOLocal(true);
1362
1363   // The pointers in the note need to be relative so that the note ends up being
1364   // placed in rodata, which is the standard location for notes.
1365   auto CreateRelPtr = [&](Constant *Ptr) {
1366     return ConstantExpr::getTrunc(
1367         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
1368                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
1369         Int32Ty);
1370   };
1371   Note->setInitializer(ConstantStruct::getAnon(
1372       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
1373        ConstantInt::get(Int32Ty, 8),                           // n_descsz
1374        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
1375        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
1376   appendToCompilerUsed(M, Note);
1377
1378   // Create a zero-length global in hwasan_globals so that the linker will
1379   // always create start and stop symbols.
1380   auto Dummy = new GlobalVariable(
1381       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
1382       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
1383   Dummy->setSection("hwasan_globals");
1384   Dummy->setComdat(NoteComdat);
1385   Dummy->setMetadata(LLVMContext::MD_associated,
1386                      MDNode::get(*C, ValueAsMetadata::get(Note)));
1387   appendToCompilerUsed(M, Dummy);
1388
1389   std::vector<GlobalVariable *> Globals;
1390   for (GlobalVariable &GV : M.globals()) {
1391     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1392         GV.isThreadLocal())
1393       continue;
1394
1395     // Common symbols can't have aliases point to them, so they can't be tagged.
1396     if (GV.hasCommonLinkage())
1397       continue;
1398
1399     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1400     // which would be broken both by adding tags and potentially by the extra
1401     // padding/alignment that we insert.
1402     if (GV.hasSection())
1403       continue;
1404
1405     Globals.push_back(&GV);
1406   }
1407
1408   MD5 Hasher;
1409   Hasher.update(M.getSourceFileName());
1410   MD5::MD5Result Hash;
1411   Hasher.final(Hash);
1412   uint8_t Tag = Hash[0];
1413
1414   for (GlobalVariable *GV : Globals) {
1415     // Skip tag 0 in order to avoid collisions with untagged memory.
1416     if (Tag == 0)
1417       Tag = 1;
1418     instrumentGlobal(GV, Tag++);
1419   }
1420 }
1421
1422 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1423   // We need to untag stack frames as we unwind past them. That is the job of
1424   // the personality function wrapper, which either wraps an existing
1425   // personality function or acts as a personality function on its own. Each
1426   // function that has a personality function or that can be unwound past has
1427   // its personality function changed to a thunk that calls the personality
1428   // function wrapper in the runtime.
1429   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1430   for (Function &F : M) {
1431     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1432       continue;
1433
1434     if (F.hasPersonalityFn()) {
1435       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1436     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1437       PersonalityFns[nullptr].push_back(&F);
1438     }
1439   }
1440
1441   if (PersonalityFns.empty())
1442     return;
1443
1444   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1445       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1446       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1447   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1448   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1449
1450   for (auto &P : PersonalityFns) {
1451     std::string ThunkName = kHwasanPersonalityThunkName;
1452     if (P.first)
1453       ThunkName += ("." + P.first->getName()).str();
1454     FunctionType *ThunkFnTy = FunctionType::get(
1455         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1456     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1457                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1458     auto *ThunkFn = Function::Create(ThunkFnTy,
1459                                      IsLocal ? GlobalValue::InternalLinkage
1460                                              : GlobalValue::LinkOnceODRLinkage,
1461                                      ThunkName, &M);
1462     if (!IsLocal) {
1463       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1464       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1465     }
1466
1467     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1468     IRBuilder<> IRB(BB);
1469     CallInst *WrapperCall = IRB.CreateCall(
1470         HwasanPersonalityWrapper,
1471         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1472          ThunkFn->getArg(3), ThunkFn->getArg(4),
1473          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1474                  : Constant::getNullValue(Int8PtrTy),
1475          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1476          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1477     WrapperCall->setTailCall();
1478     IRB.CreateRet(WrapperCall);
1479
1480     for (Function *F : P.second)
1481       F->setPersonalityFn(ThunkFn);
1482   }
1483 }
1484
1485 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1486   Scale = kDefaultShadowScale;
1487   if (ClMappingOffset.getNumOccurrences() > 0) {
1488     InGlobal = false;
1489     InTls = false;
1490     Offset = ClMappingOffset;
1491   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1492     InGlobal = false;
1493     InTls = false;
1494     Offset = 0;
1495   } else if (ClWithIfunc) {
1496     InGlobal = true;
1497     InTls = false;
1498     Offset = kDynamicShadowSentinel;
1499   } else if (ClWithTls) {
1500     InGlobal = false;
1501     InTls = true;
1502     Offset = kDynamicShadowSentinel;
1503   } else {
1504     InGlobal = false;
1505     InTls = false;
1506     Offset = kDynamicShadowSentinel;
1507   }
1508 }