lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

   1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This file is a part of HWAddressSanitizer, an address sanity checker
  11 /// based on tagged addressing.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/ADT/SmallVector.h"
  15 #include "llvm/ADT/StringExtras.h"
  16 #include "llvm/ADT/StringRef.h"
  17 #include "llvm/ADT/Triple.h"
  18 #include "llvm/IR/Attributes.h"
  19 #include "llvm/IR/BasicBlock.h"
  20 #include "llvm/IR/Constant.h"
  21 #include "llvm/IR/Constants.h"
  22 #include "llvm/IR/DataLayout.h"
  23 #include "llvm/IR/DerivedTypes.h"
  24 #include "llvm/IR/Function.h"
  25 #include "llvm/IR/IRBuilder.h"
  26 #include "llvm/IR/InlineAsm.h"
  27 #include "llvm/IR/InstVisitor.h"
  28 #include "llvm/IR/Instruction.h"
  29 #include "llvm/IR/Instructions.h"
  30 #include "llvm/IR/IntrinsicInst.h"
  31 #include "llvm/IR/Intrinsics.h"
  32 #include "llvm/IR/LLVMContext.h"
  33 #include "llvm/IR/MDBuilder.h"
  34 #include "llvm/IR/Module.h"
  35 #include "llvm/IR/Type.h"
  36 #include "llvm/IR/Value.h"
  37 #include "llvm/Pass.h"
  38 #include "llvm/Support/Casting.h"
  39 #include "llvm/Support/CommandLine.h"
  40 #include "llvm/Support/Debug.h"
  41 #include "llvm/Support/raw_ostream.h"
  42 #include "llvm/Transforms/Instrumentation.h"
  43 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  44 #include "llvm/Transforms/Utils/ModuleUtils.h"
  45 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
  46 #include <sstream>
  47
  48 using namespace llvm;
  49
  50 #define DEBUG_TYPE "hwasan"
  51
  52 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
  53 static const char *const kHwasanInitName = "__hwasan_init";
  54
  55 static const char *const kHwasanShadowMemoryDynamicAddress =
  56     "__hwasan_shadow_memory_dynamic_address";
  57
  58 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
  59 static const size_t kNumberOfAccessSizes = 5;
  60
  61 static const size_t kDefaultShadowScale = 4;
  62 static const uint64_t kDynamicShadowSentinel =
  63     std::numeric_limits<uint64_t>::max();
  64 static const unsigned kPointerTagShift = 56;
  65
  66 static const unsigned kShadowBaseAlignment = 32;
  67
  68 static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
  69     "hwasan-memory-access-callback-prefix",
  70     cl::desc("Prefix for memory access callbacks"), cl::Hidden,
  71     cl::init("__hwasan_"));
  72
  73 static cl::opt<bool>
  74     ClInstrumentWithCalls("hwasan-instrument-with-calls",
  75                 cl::desc("instrument reads and writes with callbacks"),
  76                 cl::Hidden, cl::init(false));
  77
  78 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
  79                                        cl::desc("instrument read instructions"),
  80                                        cl::Hidden, cl::init(true));
  81
  82 static cl::opt<bool> ClInstrumentWrites(
  83     "hwasan-instrument-writes", cl::desc("instrument write instructions"),
  84     cl::Hidden, cl::init(true));
  85
  86 static cl::opt<bool> ClInstrumentAtomics(
  87     "hwasan-instrument-atomics",
  88     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
  89     cl::init(true));
  90
  91 static cl::opt<bool> ClRecover(
  92     "hwasan-recover",
  93     cl::desc("Enable recovery mode (continue-after-error)."),
  94     cl::Hidden, cl::init(false));
  95
  96 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
  97                                        cl::desc("instrument stack (allocas)"),
  98                                        cl::Hidden, cl::init(true));
  99
 100 static cl::opt<bool> ClUARRetagToZero(
 101     "hwasan-uar-retag-to-zero",
 102     cl::desc("Clear alloca tags before returning from the function to allow "
 103              "non-instrumented and instrumented function calls mix. When set "
 104              "to false, allocas are retagged before returning from the "
 105              "function to detect use after return."),
 106     cl::Hidden, cl::init(true));
 107
 108 static cl::opt<bool> ClGenerateTagsWithCalls(
 109     "hwasan-generate-tags-with-calls",
 110     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
 111     cl::init(false));
 112
 113 static cl::opt<int> ClMatchAllTag(
 114     "hwasan-match-all-tag",
 115     cl::desc("don't report bad accesses via pointers with this tag"),
 116     cl::Hidden, cl::init(-1));
 117
 118 static cl::opt<bool> ClEnableKhwasan(
 119     "hwasan-kernel",
 120     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
 121     cl::Hidden, cl::init(false));
 122
 123 // These flags allow to change the shadow mapping and control how shadow memory
 124 // is accessed. The shadow mapping looks like:
 125 //    Shadow = (Mem >> scale) + offset
 126
 127 static cl::opt<unsigned long long> ClMappingOffset(
 128     "hwasan-mapping-offset",
 129     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), cl::Hidden,
 130     cl::init(0));
 131
 132 static cl::opt<bool>
 133     ClWithIfunc("hwasan-with-ifunc",
 134                 cl::desc("Access dynamic shadow through an ifunc global on "
 135                          "platforms that support this"),
 136                 cl::Hidden, cl::init(false));
 137
 138 static cl::opt<bool> ClWithTls(
 139     "hwasan-with-tls",
 140     cl::desc("Access dynamic shadow through an thread-local pointer on "
 141              "platforms that support this"),
 142     cl::Hidden, cl::init(true));
 143
 144 static cl::opt<bool>
 145     ClRecordStackHistory("hwasan-record-stack-history",
 146                          cl::desc("Record stack frames with tagged allocations "
 147                                   "in a thread-local ring buffer"),
 148                          cl::Hidden, cl::init(true));
 149 static cl::opt<bool>
 150     ClCreateFrameDescriptions("hwasan-create-frame-descriptions",
 151                               cl::desc("create static frame descriptions"),
 152                               cl::Hidden, cl::init(true));
 153
 154 static cl::opt<bool>
 155     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
 156                               cl::desc("instrument memory intrinsics"),
 157                               cl::Hidden, cl::init(true));
 158
 159 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
 160                                        cl::desc("inline all checks"),
 161                                        cl::Hidden, cl::init(false));
 162
 163 static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
 164                                   cl::desc("allow the use of ifunc"),
 165                                   cl::Hidden, cl::init(false));
 166
 167 namespace {
 168
 169 /// An instrumentation pass implementing detection of addressability bugs
 170 /// using tagged pointers.
 171 class HWAddressSanitizer : public FunctionPass {
 172 public:
 173   // Pass identification, replacement for typeid.
 174   static char ID;
 175
 176   explicit HWAddressSanitizer(bool CompileKernel = false, bool Recover = false)
 177       : FunctionPass(ID) {
 178     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
 179     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ?
 180         ClEnableKhwasan : CompileKernel;
 181   }
 182
 183   StringRef getPassName() const override { return "HWAddressSanitizer"; }
 184
 185   bool runOnFunction(Function &F) override;
 186   bool doInitialization(Module &M) override;
 187
 188   void initializeCallbacks(Module &M);
 189
 190   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
 191   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 192
 193   void untagPointerOperand(Instruction *I, Value *Addr);
 194   Value *shadowBase();
 195   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
 196   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 197                                  unsigned AccessSizeIndex,
 198                                  Instruction *InsertBefore);
 199   void instrumentMemIntrinsic(MemIntrinsic *MI);
 200   bool instrumentMemAccess(Instruction *I);
 201   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
 202                                    uint64_t *TypeSize, unsigned *Alignment,
 203                                    Value **MaybeMask);
 204
 205   bool isInterestingAlloca(const AllocaInst &AI);
 206   bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag);
 207   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
 208   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
 209   bool instrumentStack(SmallVectorImpl<AllocaInst *> &Allocas,
 210                        SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
 211   Value *getNextTagWithCall(IRBuilder<> &IRB);
 212   Value *getStackBaseTag(IRBuilder<> &IRB);
 213   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
 214                      unsigned AllocaNo);
 215   Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
 216
 217   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
 218   Value *emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
 219
 220 private:
 221   LLVMContext *C;
 222   std::string CurModuleUniqueId;
 223   Triple TargetTriple;
 224   FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
 225
 226   // Frame description is a way to pass names/sizes of local variables
 227   // to the run-time w/o adding extra executable code in every function.
 228   // We do this by creating a separate section with {PC,Descr} pairs and passing
 229   // the section beg/end to __hwasan_init_frames() at module init time.
 230   std::string createFrameString(ArrayRef<AllocaInst*> Allocas);
 231   void createFrameGlobal(Function &F, const std::string &FrameString);
 232   // Get the section name for frame descriptions. Currently ELF-only.
 233   const char *getFrameSection() { return "__hwasan_frames"; }
 234   const char *getFrameSectionBeg() { return  "__start___hwasan_frames"; }
 235   const char *getFrameSectionEnd() { return  "__stop___hwasan_frames"; }
 236   GlobalVariable *createFrameSectionBound(Module &M, Type *Ty,
 237                                           const char *Name) {
 238     auto GV = new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
 239                                  nullptr, Name);
 240     GV->setVisibility(GlobalValue::HiddenVisibility);
 241     return GV;
 242   }
 243
 244   /// This struct defines the shadow mapping using the rule:
 245   ///   shadow = (mem >> Scale) + Offset.
 246   /// If InGlobal is true, then
 247   ///   extern char __hwasan_shadow[];
 248   ///   shadow = (mem >> Scale) + &__hwasan_shadow
 249   /// If InTls is true, then
 250   ///   extern char *__hwasan_tls;
 251   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
 252   struct ShadowMapping {
 253     int Scale;
 254     uint64_t Offset;
 255     bool InGlobal;
 256     bool InTls;
 257
 258     void init(Triple &TargetTriple);
 259     unsigned getAllocaAlignment() const { return 1U << Scale; }
 260   };
 261   ShadowMapping Mapping;
 262
 263   Type *IntptrTy;
 264   Type *Int8PtrTy;
 265   Type *Int8Ty;
 266   Type *Int32Ty;
 267
 268   bool CompileKernel;
 269   bool Recover;
 270
 271   Function *HwasanCtorFunction;
 272
 273   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
 274   FunctionCallee HwasanMemoryAccessCallbackSized[2];
 275
 276   FunctionCallee HwasanTagMemoryFunc;
 277   FunctionCallee HwasanGenerateTagFunc;
 278   FunctionCallee HwasanThreadEnterFunc;
 279
 280   Constant *ShadowGlobal;
 281
 282   Value *LocalDynamicShadow = nullptr;
 283   GlobalValue *ThreadPtrGlobal = nullptr;
 284 };
 285
 286 } // end anonymous namespace
 287
 288 char HWAddressSanitizer::ID = 0;
 289
 290 INITIALIZE_PASS_BEGIN(
 291     HWAddressSanitizer, "hwasan",
 292     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 293     false)
 294 INITIALIZE_PASS_END(
 295     HWAddressSanitizer, "hwasan",
 296     "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
 297     false)
 298
 299 FunctionPass *llvm::createHWAddressSanitizerPass(bool CompileKernel,
 300                                                  bool Recover) {
 301   assert(!CompileKernel || Recover);
 302   return new HWAddressSanitizer(CompileKernel, Recover);
 303 }
 304
 305 /// Module-level initialization.
 306 ///
 307 /// inserts a call to __hwasan_init to the module's constructor list.
 308 bool HWAddressSanitizer::doInitialization(Module &M) {
 309   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
 310   auto &DL = M.getDataLayout();
 311
 312   TargetTriple = Triple(M.getTargetTriple());
 313
 314   Mapping.init(TargetTriple);
 315
 316   C = &(M.getContext());
 317   CurModuleUniqueId = getUniqueModuleId(&M);
 318   IRBuilder<> IRB(*C);
 319   IntptrTy = IRB.getIntPtrTy(DL);
 320   Int8PtrTy = IRB.getInt8PtrTy();
 321   Int8Ty = IRB.getInt8Ty();
 322   Int32Ty = IRB.getInt32Ty();
 323
 324   HwasanCtorFunction = nullptr;
 325   if (!CompileKernel) {
 326     std::tie(HwasanCtorFunction, std::ignore) =
 327         createSanitizerCtorAndInitFunctions(M, kHwasanModuleCtorName,
 328                                             kHwasanInitName,
 329                                             /*InitArgTypes=*/{},
 330                                             /*InitArgs=*/{});
 331     Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
 332     HwasanCtorFunction->setComdat(CtorComdat);
 333     appendToGlobalCtors(M, HwasanCtorFunction, 0, HwasanCtorFunction);
 334
 335     // Create a zero-length global in __hwasan_frame so that the linker will
 336     // always create start and stop symbols.
 337     //
 338     // N.B. If we ever start creating associated metadata in this pass this
 339     // global will need to be associated with the ctor.
 340     Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
 341     auto GV =
 342         new GlobalVariable(M, Int8Arr0Ty, /*isConstantGlobal*/ true,
 343                            GlobalVariable::PrivateLinkage,
 344                            Constant::getNullValue(Int8Arr0Ty), "__hwasan");
 345     GV->setSection(getFrameSection());
 346     GV->setComdat(CtorComdat);
 347     appendToCompilerUsed(M, GV);
 348
 349     IRBuilder<> IRBCtor(HwasanCtorFunction->getEntryBlock().getTerminator());
 350     IRBCtor.CreateCall(
 351         declareSanitizerInitFunction(M, "__hwasan_init_frames",
 352                                      {Int8PtrTy, Int8PtrTy}),
 353         {createFrameSectionBound(M, Int8Ty, getFrameSectionBeg()),
 354          createFrameSectionBound(M, Int8Ty, getFrameSectionEnd())});
 355   }
 356
 357   if (!TargetTriple.isAndroid())
 358     appendToCompilerUsed(
 359         M, ThreadPtrGlobal = new GlobalVariable(
 360                M, IntptrTy, false, GlobalVariable::ExternalLinkage, nullptr,
 361                "__hwasan_tls", nullptr, GlobalVariable::InitialExecTLSModel));
 362
 363   return true;
 364 }
 365
 366 void HWAddressSanitizer::initializeCallbacks(Module &M) {
 367   IRBuilder<> IRB(*C);
 368   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
 369     const std::string TypeStr = AccessIsWrite ? "store" : "load";
 370     const std::string EndingStr = Recover ? "_noabort" : "";
 371
 372     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
 373         ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
 374         FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
 375
 376     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
 377          AccessSizeIndex++) {
 378       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
 379           M.getOrInsertFunction(
 380               ClMemoryAccessCallbackPrefix + TypeStr +
 381                   itostr(1ULL << AccessSizeIndex) + EndingStr,
 382               FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
 383     }
 384   }
 385
 386   HwasanTagMemoryFunc = M.getOrInsertFunction(
 387       "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
 388   HwasanGenerateTagFunc =
 389       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
 390
 391   ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
 392                                      ArrayType::get(IRB.getInt8Ty(), 0));
 393
 394   const std::string MemIntrinCallbackPrefix =
 395       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
 396   HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
 397                                         IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 398                                         IRB.getInt8PtrTy(), IntptrTy);
 399   HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
 400                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 401                                        IRB.getInt8PtrTy(), IntptrTy);
 402   HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
 403                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
 404                                        IRB.getInt32Ty(), IntptrTy);
 405
 406   HwasanThreadEnterFunc =
 407       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy());
 408 }
 409
 410 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
 411   // An empty inline asm with input reg == output reg.
 412   // An opaque no-op cast, basically.
 413   InlineAsm *Asm = InlineAsm::get(
 414       FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
 415       StringRef(""), StringRef("=r,0"),
 416       /*hasSideEffects=*/false);
 417   return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
 418 }
 419
 420 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
 421   // Generate code only when dynamic addressing is needed.
 422   if (Mapping.Offset != kDynamicShadowSentinel)
 423     return nullptr;
 424
 425   if (Mapping.InGlobal) {
 426     return getDynamicShadowIfunc(IRB);
 427   } else {
 428     Value *GlobalDynamicAddress =
 429         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
 430             kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
 431     return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
 432   }
 433 }
 434
 435 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I,
 436                                                      bool *IsWrite,
 437                                                      uint64_t *TypeSize,
 438                                                      unsigned *Alignment,
 439                                                      Value **MaybeMask) {
 440   // Skip memory accesses inserted by another instrumentation.
 441   if (I->getMetadata("nosanitize")) return nullptr;
 442
 443   // Do not instrument the load fetching the dynamic shadow address.
 444   if (LocalDynamicShadow == I)
 445     return nullptr;
 446
 447   Value *PtrOperand = nullptr;
 448   const DataLayout &DL = I->getModule()->getDataLayout();
 449   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
 450     if (!ClInstrumentReads) return nullptr;
 451     *IsWrite = false;
 452     *TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
 453     *Alignment = LI->getAlignment();
 454     PtrOperand = LI->getPointerOperand();
 455   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
 456     if (!ClInstrumentWrites) return nullptr;
 457     *IsWrite = true;
 458     *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
 459     *Alignment = SI->getAlignment();
 460     PtrOperand = SI->getPointerOperand();
 461   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
 462     if (!ClInstrumentAtomics) return nullptr;
 463     *IsWrite = true;
 464     *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
 465     *Alignment = 0;
 466     PtrOperand = RMW->getPointerOperand();
 467   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
 468     if (!ClInstrumentAtomics) return nullptr;
 469     *IsWrite = true;
 470     *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
 471     *Alignment = 0;
 472     PtrOperand = XCHG->getPointerOperand();
 473   }
 474
 475   if (PtrOperand) {
 476     // Do not instrument accesses from different address spaces; we cannot deal
 477     // with them.
 478     Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType());
 479     if (PtrTy->getPointerAddressSpace() != 0)
 480       return nullptr;
 481
 482     // Ignore swifterror addresses.
 483     // swifterror memory addresses are mem2reg promoted by instruction
 484     // selection. As such they cannot have regular uses like an instrumentation
 485     // function and it makes no sense to track them as memory.
 486     if (PtrOperand->isSwiftError())
 487       return nullptr;
 488   }
 489
 490   return PtrOperand;
 491 }
 492
 493 static unsigned getPointerOperandIndex(Instruction *I) {
 494   if (LoadInst *LI = dyn_cast<LoadInst>(I))
 495     return LI->getPointerOperandIndex();
 496   if (StoreInst *SI = dyn_cast<StoreInst>(I))
 497     return SI->getPointerOperandIndex();
 498   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
 499     return RMW->getPointerOperandIndex();
 500   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
 501     return XCHG->getPointerOperandIndex();
 502   report_fatal_error("Unexpected instruction");
 503   return -1;
 504 }
 505
 506 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
 507   size_t Res = countTrailingZeros(TypeSize / 8);
 508   assert(Res < kNumberOfAccessSizes);
 509   return Res;
 510 }
 511
 512 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
 513   if (TargetTriple.isAArch64())
 514     return;
 515
 516   IRBuilder<> IRB(I);
 517   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
 518   Value *UntaggedPtr =
 519       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
 520   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 521 }
 522
 523 Value *HWAddressSanitizer::shadowBase() {
 524   if (LocalDynamicShadow)
 525     return LocalDynamicShadow;
 526   return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
 527                                    Int8PtrTy);
 528 }
 529
 530 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
 531   // Mem >> Scale
 532   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
 533   if (Mapping.Offset == 0)
 534     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
 535   // (Mem >> Scale) + Offset
 536   return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
 537 }
 538
 539 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
 540                                                    unsigned AccessSizeIndex,
 541                                                    Instruction *InsertBefore) {
 542   const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
 543   IRBuilder<> IRB(InsertBefore);
 544
 545   if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
 546       TargetTriple.isOSBinFormatELF() && !Recover) {
 547     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 548     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
 549     IRB.CreateCall(
 550         Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess),
 551         {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
 552     return;
 553   }
 554
 555   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
 556   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
 557                                   IRB.getInt8Ty());
 558   Value *AddrLong = untagPointer(IRB, PtrLong);
 559   Value *Shadow = memToShadow(AddrLong, IRB);
 560   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
 561   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 562
 563   int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
 564       ClMatchAllTag : (CompileKernel ? 0xFF : -1);
 565   if (matchAllTag != -1) {
 566     Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
 567         ConstantInt::get(PtrTag->getType(), matchAllTag));
 568     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
 569   }
 570
 571   Instruction *CheckTerm =
 572       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, !Recover,
 573                                 MDBuilder(*C).createBranchWeights(1, 100000));
 574
 575   IRB.SetInsertPoint(CheckTerm);
 576   InlineAsm *Asm;
 577   switch (TargetTriple.getArch()) {
 578     case Triple::x86_64:
 579       // The signal handler will find the data address in rdi.
 580       Asm = InlineAsm::get(
 581           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 582           "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
 583           "{rdi}",
 584           /*hasSideEffects=*/true);
 585       break;
 586     case Triple::aarch64:
 587     case Triple::aarch64_be:
 588       // The signal handler will find the data address in x0.
 589       Asm = InlineAsm::get(
 590           FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
 591           "brk #" + itostr(0x900 + AccessInfo),
 592           "{x0}",
 593           /*hasSideEffects=*/true);
 594       break;
 595     default:
 596       report_fatal_error("unsupported architecture");
 597   }
 598   IRB.CreateCall(Asm, PtrLong);
 599 }
 600
 601 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
 602   IRBuilder<> IRB(MI);
 603   if (isa<MemTransferInst>(MI)) {
 604     IRB.CreateCall(
 605         isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
 606         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 607          IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
 608          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 609   } else if (isa<MemSetInst>(MI)) {
 610     IRB.CreateCall(
 611         HWAsanMemset,
 612         {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
 613          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
 614          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
 615   }
 616   MI->eraseFromParent();
 617 }
 618
 619 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
 620   LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
 621   bool IsWrite = false;
 622   unsigned Alignment = 0;
 623   uint64_t TypeSize = 0;
 624   Value *MaybeMask = nullptr;
 625
 626   if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) {
 627     instrumentMemIntrinsic(cast<MemIntrinsic>(I));
 628     return true;
 629   }
 630
 631   Value *Addr =
 632       isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
 633
 634   if (!Addr)
 635     return false;
 636
 637   if (MaybeMask)
 638     return false; //FIXME
 639
 640   IRBuilder<> IRB(I);
 641   if (isPowerOf2_64(TypeSize) &&
 642       (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
 643       (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
 644        Alignment >= TypeSize / 8)) {
 645     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
 646     if (ClInstrumentWithCalls) {
 647       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
 648                      IRB.CreatePointerCast(Addr, IntptrTy));
 649     } else {
 650       instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I);
 651     }
 652   } else {
 653     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
 654                    {IRB.CreatePointerCast(Addr, IntptrTy),
 655                     ConstantInt::get(IntptrTy, TypeSize / 8)});
 656   }
 657   untagPointerOperand(I, Addr);
 658
 659   return true;
 660 }
 661
 662 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
 663   uint64_t ArraySize = 1;
 664   if (AI.isArrayAllocation()) {
 665     const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
 666     assert(CI && "non-constant array size");
 667     ArraySize = CI->getZExtValue();
 668   }
 669   Type *Ty = AI.getAllocatedType();
 670   uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
 671   return SizeInBytes * ArraySize;
 672 }
 673
 674 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
 675                                    Value *Tag) {
 676   size_t Size = (getAllocaSizeInBytes(*AI) + Mapping.getAllocaAlignment() - 1) &
 677                 ~(Mapping.getAllocaAlignment() - 1);
 678
 679   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
 680   if (ClInstrumentWithCalls) {
 681     IRB.CreateCall(HwasanTagMemoryFunc,
 682                    {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
 683                     ConstantInt::get(IntptrTy, Size)});
 684   } else {
 685     size_t ShadowSize = Size >> Mapping.Scale;
 686     Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
 687     // If this memset is not inlined, it will be intercepted in the hwasan
 688     // runtime library. That's OK, because the interceptor skips the checks if
 689     // the address is in the shadow region.
 690     // FIXME: the interceptor is not as fast as real memset. Consider lowering
 691     // llvm.memset right here into either a sequence of stores, or a call to
 692     // hwasan_tag_memory.
 693     IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, /*Align=*/1);
 694   }
 695   return true;
 696 }
 697
 698 static unsigned RetagMask(unsigned AllocaNo) {
 699   // A list of 8-bit numbers that have at most one run of non-zero bits.
 700   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
 701   // masks.
 702   // The list does not include the value 255, which is used for UAR.
 703   static unsigned FastMasks[] = {
 704       0,   1,   2,   3,   4,   6,   7,   8,   12,  14,  15, 16,  24,
 705       28,  30,  31,  32,  48,  56,  60,  62,  63,  64,  96, 112, 120,
 706       124, 126, 127, 128, 192, 224, 240, 248, 252, 254};
 707   return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
 708 }
 709
 710 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
 711   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
 712 }
 713
 714 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
 715   if (ClGenerateTagsWithCalls)
 716     return getNextTagWithCall(IRB);
 717   // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
 718   // first).
 719   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 720   auto GetStackPointerFn =
 721       Intrinsic::getDeclaration(M, Intrinsic::frameaddress);
 722   Value *StackPointer = IRB.CreateCall(
 723       GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
 724
 725   // Extract some entropy from the stack pointer for the tags.
 726   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
 727   // between functions).
 728   Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
 729   Value *StackTag =
 730       IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
 731                     "hwasan.stack.base.tag");
 732   return StackTag;
 733 }
 734
 735 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
 736                                         AllocaInst *AI, unsigned AllocaNo) {
 737   if (ClGenerateTagsWithCalls)
 738     return getNextTagWithCall(IRB);
 739   return IRB.CreateXor(StackTag,
 740                        ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
 741 }
 742
 743 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
 744   if (ClUARRetagToZero)
 745     return ConstantInt::get(IntptrTy, 0);
 746   if (ClGenerateTagsWithCalls)
 747     return getNextTagWithCall(IRB);
 748   return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
 749 }
 750
 751 // Add a tag to an address.
 752 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
 753                                       Value *PtrLong, Value *Tag) {
 754   Value *TaggedPtrLong;
 755   if (CompileKernel) {
 756     // Kernel addresses have 0xFF in the most significant byte.
 757     Value *ShiftedTag = IRB.CreateOr(
 758         IRB.CreateShl(Tag, kPointerTagShift),
 759         ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
 760     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
 761   } else {
 762     // Userspace can simply do OR (tag << 56);
 763     Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
 764     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
 765   }
 766   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
 767 }
 768
 769 // Remove tag from an address.
 770 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
 771   Value *UntaggedPtrLong;
 772   if (CompileKernel) {
 773     // Kernel addresses have 0xFF in the most significant byte.
 774     UntaggedPtrLong = IRB.CreateOr(PtrLong,
 775         ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift));
 776   } else {
 777     // Userspace addresses have 0x00.
 778     UntaggedPtrLong = IRB.CreateAnd(PtrLong,
 779         ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
 780   }
 781   return UntaggedPtrLong;
 782 }
 783
 784 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
 785   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
 786   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
 787     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
 788     // in Bionic's libc/private/bionic_tls.h.
 789     Function *ThreadPointerFunc =
 790         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
 791     Value *SlotPtr = IRB.CreatePointerCast(
 792         IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
 793                                IRB.CreateCall(ThreadPointerFunc), 0x30),
 794         Ty->getPointerTo(0));
 795     return SlotPtr;
 796   }
 797   if (ThreadPtrGlobal)
 798     return ThreadPtrGlobal;
 799
 800
 801   return nullptr;
 802 }
 803
 804 // Creates a string with a description of the stack frame (set of Allocas).
 805 // The string is intended to be human readable.
 806 // The current form is: Size1 Name1; Size2 Name2; ...
 807 std::string
 808 HWAddressSanitizer::createFrameString(ArrayRef<AllocaInst *> Allocas) {
 809   std::ostringstream Descr;
 810   for (auto AI : Allocas)
 811     Descr << getAllocaSizeInBytes(*AI) << " " <<  AI->getName().str() << "; ";
 812   return Descr.str();
 813 }
 814
 815 // Creates a global in the frame section which consists of two pointers:
 816 // the function PC and the frame string constant.
 817 void HWAddressSanitizer::createFrameGlobal(Function &F,
 818                                            const std::string &FrameString) {
 819   Module &M = *F.getParent();
 820   auto DescrGV = createPrivateGlobalForString(M, FrameString, true);
 821   auto PtrPairTy = StructType::get(F.getType(), DescrGV->getType());
 822   auto GV = new GlobalVariable(
 823       M, PtrPairTy, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
 824       ConstantStruct::get(PtrPairTy, (Constant *)&F, (Constant *)DescrGV),
 825       "__hwasan");
 826   GV->setSection(getFrameSection());
 827   appendToCompilerUsed(M, GV);
 828   // Put GV into the F's Comadat so that if F is deleted GV can be deleted too.
 829   if (auto Comdat =
 830           GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId))
 831     GV->setComdat(Comdat);
 832 }
 833
 834 Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
 835                                         bool WithFrameRecord) {
 836   if (!Mapping.InTls)
 837     return getDynamicShadowNonTls(IRB);
 838
 839   if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
 840     return getDynamicShadowIfunc(IRB);
 841
 842   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
 843   assert(SlotPtr);
 844
 845   Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 846
 847   Function *F = IRB.GetInsertBlock()->getParent();
 848   if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
 849     Value *ThreadLongEqZero =
 850         IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
 851     auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
 852         ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
 853         false, MDBuilder(*C).createBranchWeights(1, 100000)));
 854
 855     IRB.SetInsertPoint(Br);
 856     // FIXME: This should call a new runtime function with a custom calling
 857     // convention to avoid needing to spill all arguments here.
 858     IRB.CreateCall(HwasanThreadEnterFunc);
 859     LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
 860
 861     IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin());
 862     PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2);
 863     ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent());
 864     ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent());
 865     ThreadLong = ThreadLongPhi;
 866   }
 867
 868   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
 869   Value *ThreadLongMaybeUntagged =
 870       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
 871
 872   if (WithFrameRecord) {
 873     // Prepare ring buffer data.
 874     auto PC = IRB.CreatePtrToInt(F, IntptrTy);
 875     auto GetStackPointerFn =
 876         Intrinsic::getDeclaration(F->getParent(), Intrinsic::frameaddress);
 877     Value *SP = IRB.CreatePtrToInt(
 878         IRB.CreateCall(GetStackPointerFn,
 879                        {Constant::getNullValue(IRB.getInt32Ty())}),
 880         IntptrTy);
 881     // Mix SP and PC. TODO: also add the tag to the mix.
 882     // Assumptions:
 883     // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
 884     // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
 885     // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
 886     //       0xSSSSPPPPPPPPPPPP
 887     SP = IRB.CreateShl(SP, 44);
 888
 889     // Store data to ring buffer.
 890     Value *RecordPtr =
 891         IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
 892     IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
 893
 894     // Update the ring buffer. Top byte of ThreadLong defines the size of the
 895     // buffer in pages, it must be a power of two, and the start of the buffer
 896     // must be aligned by twice that much. Therefore wrap around of the ring
 897     // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
 898     // The use of AShr instead of LShr is due to
 899     //   https://bugs.llvm.org/show_bug.cgi?id=39030
 900     // Runtime library makes sure not to use the highest bit.
 901     Value *WrapMask = IRB.CreateXor(
 902         IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
 903         ConstantInt::get(IntptrTy, (uint64_t)-1));
 904     Value *ThreadLongNew = IRB.CreateAnd(
 905         IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
 906     IRB.CreateStore(ThreadLongNew, SlotPtr);
 907   }
 908
 909   // Get shadow base address by aligning RecordPtr up.
 910   // Note: this is not correct if the pointer is already aligned.
 911   // Runtime library will make sure this never happens.
 912   Value *ShadowBase = IRB.CreateAdd(
 913       IRB.CreateOr(
 914           ThreadLongMaybeUntagged,
 915           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
 916       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
 917   ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
 918   return ShadowBase;
 919 }
 920
 921 bool HWAddressSanitizer::instrumentStack(
 922     SmallVectorImpl<AllocaInst *> &Allocas,
 923     SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
 924   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
 925   // alloca addresses using that. Unfortunately, offsets are not known yet
 926   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
 927   // temp, shift-OR it into each alloca address and xor with the retag mask.
 928   // This generates one extra instruction per alloca use.
 929   for (unsigned N = 0; N < Allocas.size(); ++N) {
 930     auto *AI = Allocas[N];
 931     IRBuilder<> IRB(AI->getNextNode());
 932
 933     // Replace uses of the alloca with tagged address.
 934     Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
 935     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
 936     Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
 937     std::string Name =
 938         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
 939     Replacement->setName(Name + ".hwasan");
 940
 941     for (auto UI = AI->use_begin(), UE = AI->use_end(); UI != UE;) {
 942       Use &U = *UI++;
 943       if (U.getUser() != AILong)
 944         U.set(Replacement);
 945     }
 946
 947     tagAlloca(IRB, AI, Tag);
 948
 949     for (auto RI : RetVec) {
 950       IRB.SetInsertPoint(RI);
 951
 952       // Re-tag alloca memory with the special UAR tag.
 953       Value *Tag = getUARTag(IRB, StackTag);
 954       tagAlloca(IRB, AI, Tag);
 955     }
 956   }
 957
 958   return true;
 959 }
 960
 961 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
 962   return (AI.getAllocatedType()->isSized() &&
 963           // FIXME: instrument dynamic allocas, too
 964           AI.isStaticAlloca() &&
 965           // alloca() may be called with 0 size, ignore it.
 966           getAllocaSizeInBytes(AI) > 0 &&
 967           // We are only interested in allocas not promotable to registers.
 968           // Promotable allocas are common under -O0.
 969           !isAllocaPromotable(&AI) &&
 970           // inalloca allocas are not treated as static, and we don't want
 971           // dynamic alloca instrumentation for them as well.
 972           !AI.isUsedWithInAlloca() &&
 973           // swifterror allocas are register promoted by ISel
 974           !AI.isSwiftError());
 975 }
 976
 977 bool HWAddressSanitizer::runOnFunction(Function &F) {
 978   if (&F == HwasanCtorFunction)
 979     return false;
 980
 981   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
 982     return false;
 983
 984   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
 985
 986   SmallVector<Instruction*, 16> ToInstrument;
 987   SmallVector<AllocaInst*, 8> AllocasToInstrument;
 988   SmallVector<Instruction*, 8> RetVec;
 989   for (auto &BB : F) {
 990     for (auto &Inst : BB) {
 991       if (ClInstrumentStack)
 992         if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
 993           // Realign all allocas. We don't want small uninteresting allocas to
 994           // hide in instrumented alloca's padding.
 995           if (AI->getAlignment() < Mapping.getAllocaAlignment())
 996             AI->setAlignment(Mapping.getAllocaAlignment());
 997           // Instrument some of them.
 998           if (isInterestingAlloca(*AI))
 999             AllocasToInstrument.push_back(AI);
1000           continue;
1001         }
1002
1003       if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1004           isa<CleanupReturnInst>(Inst))
1005         RetVec.push_back(&Inst);
1006
1007       Value *MaybeMask = nullptr;
1008       bool IsWrite;
1009       unsigned Alignment;
1010       uint64_t TypeSize;
1011       Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize,
1012                                               &Alignment, &MaybeMask);
1013       if (Addr || isa<MemIntrinsic>(Inst))
1014         ToInstrument.push_back(&Inst);
1015     }
1016   }
1017
1018   if (AllocasToInstrument.empty() && ToInstrument.empty())
1019     return false;
1020
1021   if (ClCreateFrameDescriptions && !AllocasToInstrument.empty())
1022     createFrameGlobal(F, createFrameString(AllocasToInstrument));
1023
1024   initializeCallbacks(*F.getParent());
1025
1026   assert(!LocalDynamicShadow);
1027
1028   Instruction *InsertPt = &*F.getEntryBlock().begin();
1029   IRBuilder<> EntryIRB(InsertPt);
1030   LocalDynamicShadow = emitPrologue(EntryIRB,
1031                                     /*WithFrameRecord*/ ClRecordStackHistory &&
1032                                         !AllocasToInstrument.empty());
1033
1034   bool Changed = false;
1035   if (!AllocasToInstrument.empty()) {
1036     Value *StackTag =
1037         ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1038     Changed |= instrumentStack(AllocasToInstrument, RetVec, StackTag);
1039   }
1040
1041   // If we split the entry block, move any allocas that were originally in the
1042   // entry block back into the entry block so that they aren't treated as
1043   // dynamic allocas.
1044   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1045     InsertPt = &*F.getEntryBlock().begin();
1046     for (auto II = EntryIRB.GetInsertBlock()->begin(),
1047               IE = EntryIRB.GetInsertBlock()->end();
1048          II != IE;) {
1049       Instruction *I = &*II++;
1050       if (auto *AI = dyn_cast<AllocaInst>(I))
1051         if (isa<ConstantInt>(AI->getArraySize()))
1052           I->moveBefore(InsertPt);
1053     }
1054   }
1055
1056   for (auto Inst : ToInstrument)
1057     Changed |= instrumentMemAccess(Inst);
1058
1059   LocalDynamicShadow = nullptr;
1060
1061   return Changed;
1062 }
1063
1064 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) {
1065   Scale = kDefaultShadowScale;
1066   if (ClMappingOffset.getNumOccurrences() > 0) {
1067     InGlobal = false;
1068     InTls = false;
1069     Offset = ClMappingOffset;
1070   } else if (ClEnableKhwasan || ClInstrumentWithCalls) {
1071     InGlobal = false;
1072     InTls = false;
1073     Offset = 0;
1074   } else if (ClWithIfunc) {
1075     InGlobal = true;
1076     InTls = false;
1077     Offset = kDynamicShadowSentinel;
1078   } else if (ClWithTls) {
1079     InGlobal = false;
1080     InTls = true;
1081     Offset = kDynamicShadowSentinel;
1082   } else {
1083     InGlobal = false;
1084     InTls = false;
1085     Offset = kDynamicShadowSentinel;
1086   }
1087 }