1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/BinaryFormat/ELF.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/InlineAsm.h"
31 #include "llvm/IR/InstVisitor.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/Pass.h"
42 #include "llvm/Support/Casting.h"
43 #include "llvm/Support/CommandLine.h"
44 #include "llvm/Support/Debug.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include "llvm/Transforms/Instrumentation.h"
47 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
48 #include "llvm/Transforms/Utils/ModuleUtils.h"
49 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
54 #define DEBUG_TYPE "hwasan"
56 static const char *const kHwasanModuleCtorName
= "hwasan.module_ctor";
57 static const char *const kHwasanNoteName
= "hwasan.note";
58 static const char *const kHwasanInitName
= "__hwasan_init";
59 static const char *const kHwasanPersonalityThunkName
=
60 "__hwasan_personality_thunk";
62 static const char *const kHwasanShadowMemoryDynamicAddress
=
63 "__hwasan_shadow_memory_dynamic_address";
65 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
66 static const size_t kNumberOfAccessSizes
= 5;
68 static const size_t kDefaultShadowScale
= 4;
69 static const uint64_t kDynamicShadowSentinel
=
70 std::numeric_limits
<uint64_t>::max();
71 static const unsigned kPointerTagShift
= 56;
73 static const unsigned kShadowBaseAlignment
= 32;
75 static cl::opt
<std::string
> ClMemoryAccessCallbackPrefix(
76 "hwasan-memory-access-callback-prefix",
77 cl::desc("Prefix for memory access callbacks"), cl::Hidden
,
78 cl::init("__hwasan_"));
81 ClInstrumentWithCalls("hwasan-instrument-with-calls",
82 cl::desc("instrument reads and writes with callbacks"),
83 cl::Hidden
, cl::init(false));
85 static cl::opt
<bool> ClInstrumentReads("hwasan-instrument-reads",
86 cl::desc("instrument read instructions"),
87 cl::Hidden
, cl::init(true));
89 static cl::opt
<bool> ClInstrumentWrites(
90 "hwasan-instrument-writes", cl::desc("instrument write instructions"),
91 cl::Hidden
, cl::init(true));
93 static cl::opt
<bool> ClInstrumentAtomics(
94 "hwasan-instrument-atomics",
95 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden
,
98 static cl::opt
<bool> ClRecover(
100 cl::desc("Enable recovery mode (continue-after-error)."),
101 cl::Hidden
, cl::init(false));
103 static cl::opt
<bool> ClInstrumentStack("hwasan-instrument-stack",
104 cl::desc("instrument stack (allocas)"),
105 cl::Hidden
, cl::init(true));
107 static cl::opt
<bool> ClUARRetagToZero(
108 "hwasan-uar-retag-to-zero",
109 cl::desc("Clear alloca tags before returning from the function to allow "
110 "non-instrumented and instrumented function calls mix. When set "
111 "to false, allocas are retagged before returning from the "
112 "function to detect use after return."),
113 cl::Hidden
, cl::init(true));
115 static cl::opt
<bool> ClGenerateTagsWithCalls(
116 "hwasan-generate-tags-with-calls",
117 cl::desc("generate new tags with runtime library calls"), cl::Hidden
,
120 static cl::opt
<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
121 cl::Hidden
, cl::init(false));
123 static cl::opt
<int> ClMatchAllTag(
124 "hwasan-match-all-tag",
125 cl::desc("don't report bad accesses via pointers with this tag"),
126 cl::Hidden
, cl::init(-1));
128 static cl::opt
<bool> ClEnableKhwasan(
130 cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
131 cl::Hidden
, cl::init(false));
133 // These flags allow to change the shadow mapping and control how shadow memory
134 // is accessed. The shadow mapping looks like:
135 // Shadow = (Mem >> scale) + offset
137 static cl::opt
<uint64_t>
138 ClMappingOffset("hwasan-mapping-offset",
139 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
140 cl::Hidden
, cl::init(0));
143 ClWithIfunc("hwasan-with-ifunc",
144 cl::desc("Access dynamic shadow through an ifunc global on "
145 "platforms that support this"),
146 cl::Hidden
, cl::init(false));
148 static cl::opt
<bool> ClWithTls(
150 cl::desc("Access dynamic shadow through an thread-local pointer on "
151 "platforms that support this"),
152 cl::Hidden
, cl::init(true));
155 ClRecordStackHistory("hwasan-record-stack-history",
156 cl::desc("Record stack frames with tagged allocations "
157 "in a thread-local ring buffer"),
158 cl::Hidden
, cl::init(true));
160 ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
161 cl::desc("instrument memory intrinsics"),
162 cl::Hidden
, cl::init(true));
165 ClInstrumentLandingPads("hwasan-instrument-landing-pads",
166 cl::desc("instrument landing pads"), cl::Hidden
,
167 cl::init(false), cl::ZeroOrMore
);
169 static cl::opt
<bool> ClUseShortGranules(
170 "hwasan-use-short-granules",
171 cl::desc("use short granules in allocas and outlined checks"), cl::Hidden
,
172 cl::init(false), cl::ZeroOrMore
);
174 static cl::opt
<bool> ClInstrumentPersonalityFunctions(
175 "hwasan-instrument-personality-functions",
176 cl::desc("instrument personality functions"), cl::Hidden
, cl::init(false),
179 static cl::opt
<bool> ClInlineAllChecks("hwasan-inline-all-checks",
180 cl::desc("inline all checks"),
181 cl::Hidden
, cl::init(false));
185 /// An instrumentation pass implementing detection of addressability bugs
186 /// using tagged pointers.
187 class HWAddressSanitizer
{
189 explicit HWAddressSanitizer(Module
&M
, bool CompileKernel
= false,
190 bool Recover
= false) : M(M
) {
191 this->Recover
= ClRecover
.getNumOccurrences() > 0 ? ClRecover
: Recover
;
192 this->CompileKernel
= ClEnableKhwasan
.getNumOccurrences() > 0 ?
193 ClEnableKhwasan
: CompileKernel
;
198 bool sanitizeFunction(Function
&F
);
199 void initializeModule();
201 void initializeCallbacks(Module
&M
);
203 Value
*getDynamicShadowIfunc(IRBuilder
<> &IRB
);
204 Value
*getDynamicShadowNonTls(IRBuilder
<> &IRB
);
206 void untagPointerOperand(Instruction
*I
, Value
*Addr
);
208 Value
*memToShadow(Value
*Shadow
, IRBuilder
<> &IRB
);
209 void instrumentMemAccessInline(Value
*Ptr
, bool IsWrite
,
210 unsigned AccessSizeIndex
,
211 Instruction
*InsertBefore
);
212 void instrumentMemIntrinsic(MemIntrinsic
*MI
);
213 bool instrumentMemAccess(Instruction
*I
);
214 Value
*isInterestingMemoryAccess(Instruction
*I
, bool *IsWrite
,
215 uint64_t *TypeSize
, unsigned *Alignment
,
218 bool isInterestingAlloca(const AllocaInst
&AI
);
219 bool tagAlloca(IRBuilder
<> &IRB
, AllocaInst
*AI
, Value
*Tag
, size_t Size
);
220 Value
*tagPointer(IRBuilder
<> &IRB
, Type
*Ty
, Value
*PtrLong
, Value
*Tag
);
221 Value
*untagPointer(IRBuilder
<> &IRB
, Value
*PtrLong
);
222 bool instrumentStack(
223 SmallVectorImpl
<AllocaInst
*> &Allocas
,
224 DenseMap
<AllocaInst
*, std::vector
<DbgDeclareInst
*>> &AllocaDeclareMap
,
225 SmallVectorImpl
<Instruction
*> &RetVec
, Value
*StackTag
);
226 Value
*readRegister(IRBuilder
<> &IRB
, StringRef Name
);
227 bool instrumentLandingPads(SmallVectorImpl
<Instruction
*> &RetVec
);
228 Value
*getNextTagWithCall(IRBuilder
<> &IRB
);
229 Value
*getStackBaseTag(IRBuilder
<> &IRB
);
230 Value
*getAllocaTag(IRBuilder
<> &IRB
, Value
*StackTag
, AllocaInst
*AI
,
232 Value
*getUARTag(IRBuilder
<> &IRB
, Value
*StackTag
);
234 Value
*getHwasanThreadSlotPtr(IRBuilder
<> &IRB
, Type
*Ty
);
235 void emitPrologue(IRBuilder
<> &IRB
, bool WithFrameRecord
);
237 void instrumentGlobal(GlobalVariable
*GV
, uint8_t Tag
);
238 void instrumentGlobals();
240 void instrumentPersonalityFunctions();
246 FunctionCallee HWAsanMemmove
, HWAsanMemcpy
, HWAsanMemset
;
247 FunctionCallee HWAsanHandleVfork
;
249 /// This struct defines the shadow mapping using the rule:
250 /// shadow = (mem >> Scale) + Offset.
251 /// If InGlobal is true, then
252 /// extern char __hwasan_shadow[];
253 /// shadow = (mem >> Scale) + &__hwasan_shadow
254 /// If InTls is true, then
255 /// extern char *__hwasan_tls;
256 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
257 struct ShadowMapping
{
263 void init(Triple
&TargetTriple
);
264 unsigned getObjectAlignment() const { return 1U << Scale
; }
266 ShadowMapping Mapping
;
268 Type
*VoidTy
= Type::getVoidTy(M
.getContext());
273 Type
*Int64Ty
= Type::getInt64Ty(M
.getContext());
277 bool UseShortGranules
;
278 bool InstrumentLandingPads
;
280 Function
*HwasanCtorFunction
;
282 FunctionCallee HwasanMemoryAccessCallback
[2][kNumberOfAccessSizes
];
283 FunctionCallee HwasanMemoryAccessCallbackSized
[2];
285 FunctionCallee HwasanTagMemoryFunc
;
286 FunctionCallee HwasanGenerateTagFunc
;
287 FunctionCallee HwasanThreadEnterFunc
;
289 Constant
*ShadowGlobal
;
291 Value
*LocalDynamicShadow
= nullptr;
292 Value
*StackBaseTag
= nullptr;
293 GlobalValue
*ThreadPtrGlobal
= nullptr;
296 class HWAddressSanitizerLegacyPass
: public FunctionPass
{
298 // Pass identification, replacement for typeid.
301 explicit HWAddressSanitizerLegacyPass(bool CompileKernel
= false,
302 bool Recover
= false)
303 : FunctionPass(ID
), CompileKernel(CompileKernel
), Recover(Recover
) {}
305 StringRef
getPassName() const override
{ return "HWAddressSanitizer"; }
307 bool doInitialization(Module
&M
) override
{
308 HWASan
= std::make_unique
<HWAddressSanitizer
>(M
, CompileKernel
, Recover
);
312 bool runOnFunction(Function
&F
) override
{
313 return HWASan
->sanitizeFunction(F
);
316 bool doFinalization(Module
&M
) override
{
322 std::unique_ptr
<HWAddressSanitizer
> HWASan
;
327 } // end anonymous namespace
329 char HWAddressSanitizerLegacyPass::ID
= 0;
331 INITIALIZE_PASS_BEGIN(
332 HWAddressSanitizerLegacyPass
, "hwasan",
333 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
336 HWAddressSanitizerLegacyPass
, "hwasan",
337 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
340 FunctionPass
*llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel
,
342 assert(!CompileKernel
|| Recover
);
343 return new HWAddressSanitizerLegacyPass(CompileKernel
, Recover
);
346 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel
, bool Recover
)
347 : CompileKernel(CompileKernel
), Recover(Recover
) {}
349 PreservedAnalyses
HWAddressSanitizerPass::run(Module
&M
,
350 ModuleAnalysisManager
&MAM
) {
351 HWAddressSanitizer
HWASan(M
, CompileKernel
, Recover
);
352 bool Modified
= false;
353 for (Function
&F
: M
)
354 Modified
|= HWASan
.sanitizeFunction(F
);
356 return PreservedAnalyses::none();
357 return PreservedAnalyses::all();
360 /// Module-level initialization.
362 /// inserts a call to __hwasan_init to the module's constructor list.
363 void HWAddressSanitizer::initializeModule() {
364 LLVM_DEBUG(dbgs() << "Init " << M
.getName() << "\n");
365 auto &DL
= M
.getDataLayout();
367 TargetTriple
= Triple(M
.getTargetTriple());
369 Mapping
.init(TargetTriple
);
371 C
= &(M
.getContext());
373 IntptrTy
= IRB
.getIntPtrTy(DL
);
374 Int8PtrTy
= IRB
.getInt8PtrTy();
375 Int8Ty
= IRB
.getInt8Ty();
376 Int32Ty
= IRB
.getInt32Ty();
378 HwasanCtorFunction
= nullptr;
380 // Older versions of Android do not have the required runtime support for
381 // short granules, global or personality function instrumentation. On other
382 // platforms we currently require using the latest version of the runtime.
384 !TargetTriple
.isAndroid() || !TargetTriple
.isAndroidVersionLT(30);
387 ClUseShortGranules
.getNumOccurrences() ? ClUseShortGranules
: NewRuntime
;
389 // If we don't have personality function support, fall back to landing pads.
390 InstrumentLandingPads
= ClInstrumentLandingPads
.getNumOccurrences()
391 ? ClInstrumentLandingPads
394 if (!CompileKernel
) {
395 std::tie(HwasanCtorFunction
, std::ignore
) =
396 getOrCreateSanitizerCtorAndInitFunctions(
397 M
, kHwasanModuleCtorName
, kHwasanInitName
,
400 // This callback is invoked when the functions are created the first
401 // time. Hook them into the global ctors list in that case:
402 [&](Function
*Ctor
, FunctionCallee
) {
403 Comdat
*CtorComdat
= M
.getOrInsertComdat(kHwasanModuleCtorName
);
404 Ctor
->setComdat(CtorComdat
);
405 appendToGlobalCtors(M
, Ctor
, 0, Ctor
);
408 bool InstrumentGlobals
=
409 ClGlobals
.getNumOccurrences() ? ClGlobals
: NewRuntime
;
410 if (InstrumentGlobals
)
413 bool InstrumentPersonalityFunctions
=
414 ClInstrumentPersonalityFunctions
.getNumOccurrences()
415 ? ClInstrumentPersonalityFunctions
417 if (InstrumentPersonalityFunctions
)
418 instrumentPersonalityFunctions();
421 if (!TargetTriple
.isAndroid()) {
422 Constant
*C
= M
.getOrInsertGlobal("__hwasan_tls", IntptrTy
, [&] {
423 auto *GV
= new GlobalVariable(M
, IntptrTy
, /*isConstant=*/false,
424 GlobalValue::ExternalLinkage
, nullptr,
425 "__hwasan_tls", nullptr,
426 GlobalVariable::InitialExecTLSModel
);
427 appendToCompilerUsed(M
, GV
);
430 ThreadPtrGlobal
= cast
<GlobalVariable
>(C
);
434 void HWAddressSanitizer::initializeCallbacks(Module
&M
) {
436 for (size_t AccessIsWrite
= 0; AccessIsWrite
<= 1; AccessIsWrite
++) {
437 const std::string TypeStr
= AccessIsWrite
? "store" : "load";
438 const std::string EndingStr
= Recover
? "_noabort" : "";
440 HwasanMemoryAccessCallbackSized
[AccessIsWrite
] = M
.getOrInsertFunction(
441 ClMemoryAccessCallbackPrefix
+ TypeStr
+ "N" + EndingStr
,
442 FunctionType::get(IRB
.getVoidTy(), {IntptrTy
, IntptrTy
}, false));
444 for (size_t AccessSizeIndex
= 0; AccessSizeIndex
< kNumberOfAccessSizes
;
446 HwasanMemoryAccessCallback
[AccessIsWrite
][AccessSizeIndex
] =
447 M
.getOrInsertFunction(
448 ClMemoryAccessCallbackPrefix
+ TypeStr
+
449 itostr(1ULL << AccessSizeIndex
) + EndingStr
,
450 FunctionType::get(IRB
.getVoidTy(), {IntptrTy
}, false));
454 HwasanTagMemoryFunc
= M
.getOrInsertFunction(
455 "__hwasan_tag_memory", IRB
.getVoidTy(), Int8PtrTy
, Int8Ty
, IntptrTy
);
456 HwasanGenerateTagFunc
=
457 M
.getOrInsertFunction("__hwasan_generate_tag", Int8Ty
);
459 ShadowGlobal
= M
.getOrInsertGlobal("__hwasan_shadow",
460 ArrayType::get(IRB
.getInt8Ty(), 0));
462 const std::string MemIntrinCallbackPrefix
=
463 CompileKernel
? std::string("") : ClMemoryAccessCallbackPrefix
;
464 HWAsanMemmove
= M
.getOrInsertFunction(MemIntrinCallbackPrefix
+ "memmove",
465 IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(),
466 IRB
.getInt8PtrTy(), IntptrTy
);
467 HWAsanMemcpy
= M
.getOrInsertFunction(MemIntrinCallbackPrefix
+ "memcpy",
468 IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(),
469 IRB
.getInt8PtrTy(), IntptrTy
);
470 HWAsanMemset
= M
.getOrInsertFunction(MemIntrinCallbackPrefix
+ "memset",
471 IRB
.getInt8PtrTy(), IRB
.getInt8PtrTy(),
472 IRB
.getInt32Ty(), IntptrTy
);
475 M
.getOrInsertFunction("__hwasan_handle_vfork", IRB
.getVoidTy(), IntptrTy
);
477 HwasanThreadEnterFunc
=
478 M
.getOrInsertFunction("__hwasan_thread_enter", IRB
.getVoidTy());
481 Value
*HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder
<> &IRB
) {
482 // An empty inline asm with input reg == output reg.
483 // An opaque no-op cast, basically.
484 InlineAsm
*Asm
= InlineAsm::get(
485 FunctionType::get(Int8PtrTy
, {ShadowGlobal
->getType()}, false),
486 StringRef(""), StringRef("=r,0"),
487 /*hasSideEffects=*/false);
488 return IRB
.CreateCall(Asm
, {ShadowGlobal
}, ".hwasan.shadow");
491 Value
*HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder
<> &IRB
) {
492 // Generate code only when dynamic addressing is needed.
493 if (Mapping
.Offset
!= kDynamicShadowSentinel
)
496 if (Mapping
.InGlobal
) {
497 return getDynamicShadowIfunc(IRB
);
499 Value
*GlobalDynamicAddress
=
500 IRB
.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
501 kHwasanShadowMemoryDynamicAddress
, Int8PtrTy
);
502 return IRB
.CreateLoad(Int8PtrTy
, GlobalDynamicAddress
);
506 Value
*HWAddressSanitizer::isInterestingMemoryAccess(Instruction
*I
,
511 // Skip memory accesses inserted by another instrumentation.
512 if (I
->hasMetadata("nosanitize")) return nullptr;
514 // Do not instrument the load fetching the dynamic shadow address.
515 if (LocalDynamicShadow
== I
)
518 Value
*PtrOperand
= nullptr;
519 const DataLayout
&DL
= I
->getModule()->getDataLayout();
520 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(I
)) {
521 if (!ClInstrumentReads
) return nullptr;
523 *TypeSize
= DL
.getTypeStoreSizeInBits(LI
->getType());
524 *Alignment
= LI
->getAlignment();
525 PtrOperand
= LI
->getPointerOperand();
526 } else if (StoreInst
*SI
= dyn_cast
<StoreInst
>(I
)) {
527 if (!ClInstrumentWrites
) return nullptr;
529 *TypeSize
= DL
.getTypeStoreSizeInBits(SI
->getValueOperand()->getType());
530 *Alignment
= SI
->getAlignment();
531 PtrOperand
= SI
->getPointerOperand();
532 } else if (AtomicRMWInst
*RMW
= dyn_cast
<AtomicRMWInst
>(I
)) {
533 if (!ClInstrumentAtomics
) return nullptr;
535 *TypeSize
= DL
.getTypeStoreSizeInBits(RMW
->getValOperand()->getType());
537 PtrOperand
= RMW
->getPointerOperand();
538 } else if (AtomicCmpXchgInst
*XCHG
= dyn_cast
<AtomicCmpXchgInst
>(I
)) {
539 if (!ClInstrumentAtomics
) return nullptr;
541 *TypeSize
= DL
.getTypeStoreSizeInBits(XCHG
->getCompareOperand()->getType());
543 PtrOperand
= XCHG
->getPointerOperand();
547 // Do not instrument accesses from different address spaces; we cannot deal
549 Type
*PtrTy
= cast
<PointerType
>(PtrOperand
->getType()->getScalarType());
550 if (PtrTy
->getPointerAddressSpace() != 0)
553 // Ignore swifterror addresses.
554 // swifterror memory addresses are mem2reg promoted by instruction
555 // selection. As such they cannot have regular uses like an instrumentation
556 // function and it makes no sense to track them as memory.
557 if (PtrOperand
->isSwiftError())
564 static unsigned getPointerOperandIndex(Instruction
*I
) {
565 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(I
))
566 return LI
->getPointerOperandIndex();
567 if (StoreInst
*SI
= dyn_cast
<StoreInst
>(I
))
568 return SI
->getPointerOperandIndex();
569 if (AtomicRMWInst
*RMW
= dyn_cast
<AtomicRMWInst
>(I
))
570 return RMW
->getPointerOperandIndex();
571 if (AtomicCmpXchgInst
*XCHG
= dyn_cast
<AtomicCmpXchgInst
>(I
))
572 return XCHG
->getPointerOperandIndex();
573 report_fatal_error("Unexpected instruction");
577 static size_t TypeSizeToSizeIndex(uint32_t TypeSize
) {
578 size_t Res
= countTrailingZeros(TypeSize
/ 8);
579 assert(Res
< kNumberOfAccessSizes
);
583 void HWAddressSanitizer::untagPointerOperand(Instruction
*I
, Value
*Addr
) {
584 if (TargetTriple
.isAArch64())
588 Value
*AddrLong
= IRB
.CreatePointerCast(Addr
, IntptrTy
);
590 IRB
.CreateIntToPtr(untagPointer(IRB
, AddrLong
), Addr
->getType());
591 I
->setOperand(getPointerOperandIndex(I
), UntaggedPtr
);
594 Value
*HWAddressSanitizer::shadowBase() {
595 if (LocalDynamicShadow
)
596 return LocalDynamicShadow
;
597 return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy
, Mapping
.Offset
),
601 Value
*HWAddressSanitizer::memToShadow(Value
*Mem
, IRBuilder
<> &IRB
) {
603 Value
*Shadow
= IRB
.CreateLShr(Mem
, Mapping
.Scale
);
604 if (Mapping
.Offset
== 0)
605 return IRB
.CreateIntToPtr(Shadow
, Int8PtrTy
);
606 // (Mem >> Scale) + Offset
607 return IRB
.CreateGEP(Int8Ty
, shadowBase(), Shadow
);
610 void HWAddressSanitizer::instrumentMemAccessInline(Value
*Ptr
, bool IsWrite
,
611 unsigned AccessSizeIndex
,
612 Instruction
*InsertBefore
) {
613 const int64_t AccessInfo
= Recover
* 0x20 + IsWrite
* 0x10 + AccessSizeIndex
;
614 IRBuilder
<> IRB(InsertBefore
);
616 if (!ClInlineAllChecks
&& TargetTriple
.isAArch64() &&
617 TargetTriple
.isOSBinFormatELF() && !Recover
) {
618 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
619 Ptr
= IRB
.CreateBitCast(Ptr
, Int8PtrTy
);
620 IRB
.CreateCall(Intrinsic::getDeclaration(
622 ? Intrinsic::hwasan_check_memaccess_shortgranules
623 : Intrinsic::hwasan_check_memaccess
),
624 {shadowBase(), Ptr
, ConstantInt::get(Int32Ty
, AccessInfo
)});
628 Value
*PtrLong
= IRB
.CreatePointerCast(Ptr
, IntptrTy
);
629 Value
*PtrTag
= IRB
.CreateTrunc(IRB
.CreateLShr(PtrLong
, kPointerTagShift
),
631 Value
*AddrLong
= untagPointer(IRB
, PtrLong
);
632 Value
*Shadow
= memToShadow(AddrLong
, IRB
);
633 Value
*MemTag
= IRB
.CreateLoad(Int8Ty
, Shadow
);
634 Value
*TagMismatch
= IRB
.CreateICmpNE(PtrTag
, MemTag
);
636 int matchAllTag
= ClMatchAllTag
.getNumOccurrences() > 0 ?
637 ClMatchAllTag
: (CompileKernel
? 0xFF : -1);
638 if (matchAllTag
!= -1) {
639 Value
*TagNotIgnored
= IRB
.CreateICmpNE(PtrTag
,
640 ConstantInt::get(PtrTag
->getType(), matchAllTag
));
641 TagMismatch
= IRB
.CreateAnd(TagMismatch
, TagNotIgnored
);
644 Instruction
*CheckTerm
=
645 SplitBlockAndInsertIfThen(TagMismatch
, InsertBefore
, false,
646 MDBuilder(*C
).createBranchWeights(1, 100000));
648 IRB
.SetInsertPoint(CheckTerm
);
649 Value
*OutOfShortGranuleTagRange
=
650 IRB
.CreateICmpUGT(MemTag
, ConstantInt::get(Int8Ty
, 15));
651 Instruction
*CheckFailTerm
=
652 SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange
, CheckTerm
, !Recover
,
653 MDBuilder(*C
).createBranchWeights(1, 100000));
655 IRB
.SetInsertPoint(CheckTerm
);
656 Value
*PtrLowBits
= IRB
.CreateTrunc(IRB
.CreateAnd(PtrLong
, 15), Int8Ty
);
657 PtrLowBits
= IRB
.CreateAdd(
658 PtrLowBits
, ConstantInt::get(Int8Ty
, (1 << AccessSizeIndex
) - 1));
659 Value
*PtrLowBitsOOB
= IRB
.CreateICmpUGE(PtrLowBits
, MemTag
);
660 SplitBlockAndInsertIfThen(PtrLowBitsOOB
, CheckTerm
, false,
661 MDBuilder(*C
).createBranchWeights(1, 100000),
662 nullptr, nullptr, CheckFailTerm
->getParent());
664 IRB
.SetInsertPoint(CheckTerm
);
665 Value
*InlineTagAddr
= IRB
.CreateOr(AddrLong
, 15);
666 InlineTagAddr
= IRB
.CreateIntToPtr(InlineTagAddr
, Int8PtrTy
);
667 Value
*InlineTag
= IRB
.CreateLoad(Int8Ty
, InlineTagAddr
);
668 Value
*InlineTagMismatch
= IRB
.CreateICmpNE(PtrTag
, InlineTag
);
669 SplitBlockAndInsertIfThen(InlineTagMismatch
, CheckTerm
, false,
670 MDBuilder(*C
).createBranchWeights(1, 100000),
671 nullptr, nullptr, CheckFailTerm
->getParent());
673 IRB
.SetInsertPoint(CheckFailTerm
);
675 switch (TargetTriple
.getArch()) {
677 // The signal handler will find the data address in rdi.
678 Asm
= InlineAsm::get(
679 FunctionType::get(IRB
.getVoidTy(), {PtrLong
->getType()}, false),
680 "int3\nnopl " + itostr(0x40 + AccessInfo
) + "(%rax)",
682 /*hasSideEffects=*/true);
684 case Triple::aarch64
:
685 case Triple::aarch64_be
:
686 // The signal handler will find the data address in x0.
687 Asm
= InlineAsm::get(
688 FunctionType::get(IRB
.getVoidTy(), {PtrLong
->getType()}, false),
689 "brk #" + itostr(0x900 + AccessInfo
),
691 /*hasSideEffects=*/true);
694 report_fatal_error("unsupported architecture");
696 IRB
.CreateCall(Asm
, PtrLong
);
698 cast
<BranchInst
>(CheckFailTerm
)->setSuccessor(0, CheckTerm
->getParent());
701 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic
*MI
) {
703 if (isa
<MemTransferInst
>(MI
)) {
705 isa
<MemMoveInst
>(MI
) ? HWAsanMemmove
: HWAsanMemcpy
,
706 {IRB
.CreatePointerCast(MI
->getOperand(0), IRB
.getInt8PtrTy()),
707 IRB
.CreatePointerCast(MI
->getOperand(1), IRB
.getInt8PtrTy()),
708 IRB
.CreateIntCast(MI
->getOperand(2), IntptrTy
, false)});
709 } else if (isa
<MemSetInst
>(MI
)) {
712 {IRB
.CreatePointerCast(MI
->getOperand(0), IRB
.getInt8PtrTy()),
713 IRB
.CreateIntCast(MI
->getOperand(1), IRB
.getInt32Ty(), false),
714 IRB
.CreateIntCast(MI
->getOperand(2), IntptrTy
, false)});
716 MI
->eraseFromParent();
719 bool HWAddressSanitizer::instrumentMemAccess(Instruction
*I
) {
720 LLVM_DEBUG(dbgs() << "Instrumenting: " << *I
<< "\n");
721 bool IsWrite
= false;
722 unsigned Alignment
= 0;
723 uint64_t TypeSize
= 0;
724 Value
*MaybeMask
= nullptr;
726 if (ClInstrumentMemIntrinsics
&& isa
<MemIntrinsic
>(I
)) {
727 instrumentMemIntrinsic(cast
<MemIntrinsic
>(I
));
732 isInterestingMemoryAccess(I
, &IsWrite
, &TypeSize
, &Alignment
, &MaybeMask
);
738 return false; //FIXME
741 if (isPowerOf2_64(TypeSize
) &&
742 (TypeSize
/ 8 <= (1UL << (kNumberOfAccessSizes
- 1))) &&
743 (Alignment
>= (1UL << Mapping
.Scale
) || Alignment
== 0 ||
744 Alignment
>= TypeSize
/ 8)) {
745 size_t AccessSizeIndex
= TypeSizeToSizeIndex(TypeSize
);
746 if (ClInstrumentWithCalls
) {
747 IRB
.CreateCall(HwasanMemoryAccessCallback
[IsWrite
][AccessSizeIndex
],
748 IRB
.CreatePointerCast(Addr
, IntptrTy
));
750 instrumentMemAccessInline(Addr
, IsWrite
, AccessSizeIndex
, I
);
753 IRB
.CreateCall(HwasanMemoryAccessCallbackSized
[IsWrite
],
754 {IRB
.CreatePointerCast(Addr
, IntptrTy
),
755 ConstantInt::get(IntptrTy
, TypeSize
/ 8)});
757 untagPointerOperand(I
, Addr
);
762 static uint64_t getAllocaSizeInBytes(const AllocaInst
&AI
) {
763 uint64_t ArraySize
= 1;
764 if (AI
.isArrayAllocation()) {
765 const ConstantInt
*CI
= dyn_cast
<ConstantInt
>(AI
.getArraySize());
766 assert(CI
&& "non-constant array size");
767 ArraySize
= CI
->getZExtValue();
769 Type
*Ty
= AI
.getAllocatedType();
770 uint64_t SizeInBytes
= AI
.getModule()->getDataLayout().getTypeAllocSize(Ty
);
771 return SizeInBytes
* ArraySize
;
774 bool HWAddressSanitizer::tagAlloca(IRBuilder
<> &IRB
, AllocaInst
*AI
,
775 Value
*Tag
, size_t Size
) {
776 size_t AlignedSize
= alignTo(Size
, Mapping
.getObjectAlignment());
777 if (!UseShortGranules
)
780 Value
*JustTag
= IRB
.CreateTrunc(Tag
, IRB
.getInt8Ty());
781 if (ClInstrumentWithCalls
) {
782 IRB
.CreateCall(HwasanTagMemoryFunc
,
783 {IRB
.CreatePointerCast(AI
, Int8PtrTy
), JustTag
,
784 ConstantInt::get(IntptrTy
, AlignedSize
)});
786 size_t ShadowSize
= Size
>> Mapping
.Scale
;
787 Value
*ShadowPtr
= memToShadow(IRB
.CreatePointerCast(AI
, IntptrTy
), IRB
);
788 // If this memset is not inlined, it will be intercepted in the hwasan
789 // runtime library. That's OK, because the interceptor skips the checks if
790 // the address is in the shadow region.
791 // FIXME: the interceptor is not as fast as real memset. Consider lowering
792 // llvm.memset right here into either a sequence of stores, or a call to
793 // hwasan_tag_memory.
795 IRB
.CreateMemSet(ShadowPtr
, JustTag
, ShadowSize
, /*Align=*/1);
796 if (Size
!= AlignedSize
) {
798 ConstantInt::get(Int8Ty
, Size
% Mapping
.getObjectAlignment()),
799 IRB
.CreateConstGEP1_32(Int8Ty
, ShadowPtr
, ShadowSize
));
800 IRB
.CreateStore(JustTag
, IRB
.CreateConstGEP1_32(
801 Int8Ty
, IRB
.CreateBitCast(AI
, Int8PtrTy
),
808 static unsigned RetagMask(unsigned AllocaNo
) {
809 // A list of 8-bit numbers that have at most one run of non-zero bits.
810 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
812 // The list does not include the value 255, which is used for UAR.
814 // Because we are more likely to use earlier elements of this list than later
815 // ones, it is sorted in increasing order of probability of collision with a
816 // mask allocated (temporally) nearby. The program that generated this list
818 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
819 static unsigned FastMasks
[] = {0, 128, 64, 192, 32, 96, 224, 112, 240,
820 48, 16, 120, 248, 56, 24, 8, 124, 252,
821 60, 28, 12, 4, 126, 254, 62, 30, 14,
822 6, 2, 127, 63, 31, 15, 7, 3, 1};
823 return FastMasks
[AllocaNo
% (sizeof(FastMasks
) / sizeof(FastMasks
[0]))];
826 Value
*HWAddressSanitizer::getNextTagWithCall(IRBuilder
<> &IRB
) {
827 return IRB
.CreateZExt(IRB
.CreateCall(HwasanGenerateTagFunc
), IntptrTy
);
830 Value
*HWAddressSanitizer::getStackBaseTag(IRBuilder
<> &IRB
) {
831 if (ClGenerateTagsWithCalls
)
832 return getNextTagWithCall(IRB
);
835 // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
837 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
838 auto GetStackPointerFn
= Intrinsic::getDeclaration(
839 M
, Intrinsic::frameaddress
,
840 IRB
.getInt8PtrTy(M
->getDataLayout().getAllocaAddrSpace()));
841 Value
*StackPointer
= IRB
.CreateCall(
842 GetStackPointerFn
, {Constant::getNullValue(IRB
.getInt32Ty())});
844 // Extract some entropy from the stack pointer for the tags.
845 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
846 // between functions).
847 Value
*StackPointerLong
= IRB
.CreatePointerCast(StackPointer
, IntptrTy
);
849 IRB
.CreateXor(StackPointerLong
, IRB
.CreateLShr(StackPointerLong
, 20),
850 "hwasan.stack.base.tag");
854 Value
*HWAddressSanitizer::getAllocaTag(IRBuilder
<> &IRB
, Value
*StackTag
,
855 AllocaInst
*AI
, unsigned AllocaNo
) {
856 if (ClGenerateTagsWithCalls
)
857 return getNextTagWithCall(IRB
);
858 return IRB
.CreateXor(StackTag
,
859 ConstantInt::get(IntptrTy
, RetagMask(AllocaNo
)));
862 Value
*HWAddressSanitizer::getUARTag(IRBuilder
<> &IRB
, Value
*StackTag
) {
863 if (ClUARRetagToZero
)
864 return ConstantInt::get(IntptrTy
, 0);
865 if (ClGenerateTagsWithCalls
)
866 return getNextTagWithCall(IRB
);
867 return IRB
.CreateXor(StackTag
, ConstantInt::get(IntptrTy
, 0xFFU
));
870 // Add a tag to an address.
871 Value
*HWAddressSanitizer::tagPointer(IRBuilder
<> &IRB
, Type
*Ty
,
872 Value
*PtrLong
, Value
*Tag
) {
873 Value
*TaggedPtrLong
;
875 // Kernel addresses have 0xFF in the most significant byte.
876 Value
*ShiftedTag
= IRB
.CreateOr(
877 IRB
.CreateShl(Tag
, kPointerTagShift
),
878 ConstantInt::get(IntptrTy
, (1ULL << kPointerTagShift
) - 1));
879 TaggedPtrLong
= IRB
.CreateAnd(PtrLong
, ShiftedTag
);
881 // Userspace can simply do OR (tag << 56);
882 Value
*ShiftedTag
= IRB
.CreateShl(Tag
, kPointerTagShift
);
883 TaggedPtrLong
= IRB
.CreateOr(PtrLong
, ShiftedTag
);
885 return IRB
.CreateIntToPtr(TaggedPtrLong
, Ty
);
888 // Remove tag from an address.
889 Value
*HWAddressSanitizer::untagPointer(IRBuilder
<> &IRB
, Value
*PtrLong
) {
890 Value
*UntaggedPtrLong
;
892 // Kernel addresses have 0xFF in the most significant byte.
893 UntaggedPtrLong
= IRB
.CreateOr(PtrLong
,
894 ConstantInt::get(PtrLong
->getType(), 0xFFULL
<< kPointerTagShift
));
896 // Userspace addresses have 0x00.
897 UntaggedPtrLong
= IRB
.CreateAnd(PtrLong
,
898 ConstantInt::get(PtrLong
->getType(), ~(0xFFULL
<< kPointerTagShift
)));
900 return UntaggedPtrLong
;
903 Value
*HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder
<> &IRB
, Type
*Ty
) {
904 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
905 if (TargetTriple
.isAArch64() && TargetTriple
.isAndroid()) {
906 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
907 // in Bionic's libc/private/bionic_tls.h.
908 Function
*ThreadPointerFunc
=
909 Intrinsic::getDeclaration(M
, Intrinsic::thread_pointer
);
910 Value
*SlotPtr
= IRB
.CreatePointerCast(
911 IRB
.CreateConstGEP1_32(IRB
.getInt8Ty(),
912 IRB
.CreateCall(ThreadPointerFunc
), 0x30),
913 Ty
->getPointerTo(0));
917 return ThreadPtrGlobal
;
923 void HWAddressSanitizer::emitPrologue(IRBuilder
<> &IRB
, bool WithFrameRecord
) {
924 if (!Mapping
.InTls
) {
925 LocalDynamicShadow
= getDynamicShadowNonTls(IRB
);
929 if (!WithFrameRecord
&& TargetTriple
.isAndroid()) {
930 LocalDynamicShadow
= getDynamicShadowIfunc(IRB
);
934 Value
*SlotPtr
= getHwasanThreadSlotPtr(IRB
, IntptrTy
);
937 Instruction
*ThreadLong
= IRB
.CreateLoad(IntptrTy
, SlotPtr
);
939 Function
*F
= IRB
.GetInsertBlock()->getParent();
940 if (F
->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
941 Value
*ThreadLongEqZero
=
942 IRB
.CreateICmpEQ(ThreadLong
, ConstantInt::get(IntptrTy
, 0));
943 auto *Br
= cast
<BranchInst
>(SplitBlockAndInsertIfThen(
944 ThreadLongEqZero
, cast
<Instruction
>(ThreadLongEqZero
)->getNextNode(),
945 false, MDBuilder(*C
).createBranchWeights(1, 100000)));
947 IRB
.SetInsertPoint(Br
);
948 // FIXME: This should call a new runtime function with a custom calling
949 // convention to avoid needing to spill all arguments here.
950 IRB
.CreateCall(HwasanThreadEnterFunc
);
951 LoadInst
*ReloadThreadLong
= IRB
.CreateLoad(IntptrTy
, SlotPtr
);
953 IRB
.SetInsertPoint(&*Br
->getSuccessor(0)->begin());
954 PHINode
*ThreadLongPhi
= IRB
.CreatePHI(IntptrTy
, 2);
955 ThreadLongPhi
->addIncoming(ThreadLong
, ThreadLong
->getParent());
956 ThreadLongPhi
->addIncoming(ReloadThreadLong
, ReloadThreadLong
->getParent());
957 ThreadLong
= ThreadLongPhi
;
960 // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
961 Value
*ThreadLongMaybeUntagged
=
962 TargetTriple
.isAArch64() ? ThreadLong
: untagPointer(IRB
, ThreadLong
);
964 if (WithFrameRecord
) {
965 StackBaseTag
= IRB
.CreateAShr(ThreadLong
, 3);
967 // Prepare ring buffer data.
969 if (TargetTriple
.getArch() == Triple::aarch64
)
970 PC
= readRegister(IRB
, "pc");
972 PC
= IRB
.CreatePtrToInt(F
, IntptrTy
);
973 Module
*M
= F
->getParent();
974 auto GetStackPointerFn
= Intrinsic::getDeclaration(
975 M
, Intrinsic::frameaddress
,
976 IRB
.getInt8PtrTy(M
->getDataLayout().getAllocaAddrSpace()));
977 Value
*SP
= IRB
.CreatePtrToInt(
978 IRB
.CreateCall(GetStackPointerFn
,
979 {Constant::getNullValue(IRB
.getInt32Ty())}),
983 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
984 // SP is 0xsssssssssssSSSS0 (4 lower bits are zero)
985 // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
986 // 0xSSSSPPPPPPPPPPPP
987 SP
= IRB
.CreateShl(SP
, 44);
989 // Store data to ring buffer.
991 IRB
.CreateIntToPtr(ThreadLongMaybeUntagged
, IntptrTy
->getPointerTo(0));
992 IRB
.CreateStore(IRB
.CreateOr(PC
, SP
), RecordPtr
);
994 // Update the ring buffer. Top byte of ThreadLong defines the size of the
995 // buffer in pages, it must be a power of two, and the start of the buffer
996 // must be aligned by twice that much. Therefore wrap around of the ring
997 // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
998 // The use of AShr instead of LShr is due to
999 // https://bugs.llvm.org/show_bug.cgi?id=39030
1000 // Runtime library makes sure not to use the highest bit.
1001 Value
*WrapMask
= IRB
.CreateXor(
1002 IRB
.CreateShl(IRB
.CreateAShr(ThreadLong
, 56), 12, "", true, true),
1003 ConstantInt::get(IntptrTy
, (uint64_t)-1));
1004 Value
*ThreadLongNew
= IRB
.CreateAnd(
1005 IRB
.CreateAdd(ThreadLong
, ConstantInt::get(IntptrTy
, 8)), WrapMask
);
1006 IRB
.CreateStore(ThreadLongNew
, SlotPtr
);
1009 // Get shadow base address by aligning RecordPtr up.
1010 // Note: this is not correct if the pointer is already aligned.
1011 // Runtime library will make sure this never happens.
1012 LocalDynamicShadow
= IRB
.CreateAdd(
1014 ThreadLongMaybeUntagged
,
1015 ConstantInt::get(IntptrTy
, (1ULL << kShadowBaseAlignment
) - 1)),
1016 ConstantInt::get(IntptrTy
, 1), "hwasan.shadow");
1017 LocalDynamicShadow
= IRB
.CreateIntToPtr(LocalDynamicShadow
, Int8PtrTy
);
1020 Value
*HWAddressSanitizer::readRegister(IRBuilder
<> &IRB
, StringRef Name
) {
1021 Module
*M
= IRB
.GetInsertBlock()->getParent()->getParent();
1022 Function
*ReadRegister
=
1023 Intrinsic::getDeclaration(M
, Intrinsic::read_register
, IntptrTy
);
1024 MDNode
*MD
= MDNode::get(*C
, {MDString::get(*C
, Name
)});
1025 Value
*Args
[] = {MetadataAsValue::get(*C
, MD
)};
1026 return IRB
.CreateCall(ReadRegister
, Args
);
1029 bool HWAddressSanitizer::instrumentLandingPads(
1030 SmallVectorImpl
<Instruction
*> &LandingPadVec
) {
1031 for (auto *LP
: LandingPadVec
) {
1032 IRBuilder
<> IRB(LP
->getNextNode());
1035 {readRegister(IRB
, (TargetTriple
.getArch() == Triple::x86_64
) ? "rsp"
1041 bool HWAddressSanitizer::instrumentStack(
1042 SmallVectorImpl
<AllocaInst
*> &Allocas
,
1043 DenseMap
<AllocaInst
*, std::vector
<DbgDeclareInst
*>> &AllocaDeclareMap
,
1044 SmallVectorImpl
<Instruction
*> &RetVec
, Value
*StackTag
) {
1045 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1046 // alloca addresses using that. Unfortunately, offsets are not known yet
1047 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1048 // temp, shift-OR it into each alloca address and xor with the retag mask.
1049 // This generates one extra instruction per alloca use.
1050 for (unsigned N
= 0; N
< Allocas
.size(); ++N
) {
1051 auto *AI
= Allocas
[N
];
1052 IRBuilder
<> IRB(AI
->getNextNode());
1054 // Replace uses of the alloca with tagged address.
1055 Value
*Tag
= getAllocaTag(IRB
, StackTag
, AI
, N
);
1056 Value
*AILong
= IRB
.CreatePointerCast(AI
, IntptrTy
);
1057 Value
*Replacement
= tagPointer(IRB
, AI
->getType(), AILong
, Tag
);
1059 AI
->hasName() ? AI
->getName().str() : "alloca." + itostr(N
);
1060 Replacement
->setName(Name
+ ".hwasan");
1062 AI
->replaceUsesWithIf(Replacement
,
1063 [AILong
](Use
&U
) { return U
.getUser() != AILong
; });
1065 for (auto *DDI
: AllocaDeclareMap
.lookup(AI
)) {
1066 DIExpression
*OldExpr
= DDI
->getExpression();
1067 DIExpression
*NewExpr
= DIExpression::append(
1068 OldExpr
, {dwarf::DW_OP_LLVM_tag_offset
, RetagMask(N
)});
1069 DDI
->setArgOperand(2, MetadataAsValue::get(*C
, NewExpr
));
1072 size_t Size
= getAllocaSizeInBytes(*AI
);
1073 tagAlloca(IRB
, AI
, Tag
, Size
);
1075 for (auto RI
: RetVec
) {
1076 IRB
.SetInsertPoint(RI
);
1078 // Re-tag alloca memory with the special UAR tag.
1079 Value
*Tag
= getUARTag(IRB
, StackTag
);
1080 tagAlloca(IRB
, AI
, Tag
, alignTo(Size
, Mapping
.getObjectAlignment()));
1087 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst
&AI
) {
1088 return (AI
.getAllocatedType()->isSized() &&
1089 // FIXME: instrument dynamic allocas, too
1090 AI
.isStaticAlloca() &&
1091 // alloca() may be called with 0 size, ignore it.
1092 getAllocaSizeInBytes(AI
) > 0 &&
1093 // We are only interested in allocas not promotable to registers.
1094 // Promotable allocas are common under -O0.
1095 !isAllocaPromotable(&AI
) &&
1096 // inalloca allocas are not treated as static, and we don't want
1097 // dynamic alloca instrumentation for them as well.
1098 !AI
.isUsedWithInAlloca() &&
1099 // swifterror allocas are register promoted by ISel
1100 !AI
.isSwiftError());
1103 bool HWAddressSanitizer::sanitizeFunction(Function
&F
) {
1104 if (&F
== HwasanCtorFunction
)
1107 if (!F
.hasFnAttribute(Attribute::SanitizeHWAddress
))
1110 LLVM_DEBUG(dbgs() << "Function: " << F
.getName() << "\n");
1112 SmallVector
<Instruction
*, 16> ToInstrument
;
1113 SmallVector
<AllocaInst
*, 8> AllocasToInstrument
;
1114 SmallVector
<Instruction
*, 8> RetVec
;
1115 SmallVector
<Instruction
*, 8> LandingPadVec
;
1116 DenseMap
<AllocaInst
*, std::vector
<DbgDeclareInst
*>> AllocaDeclareMap
;
1117 for (auto &BB
: F
) {
1118 for (auto &Inst
: BB
) {
1119 if (ClInstrumentStack
)
1120 if (AllocaInst
*AI
= dyn_cast
<AllocaInst
>(&Inst
)) {
1121 if (isInterestingAlloca(*AI
))
1122 AllocasToInstrument
.push_back(AI
);
1126 if (isa
<ReturnInst
>(Inst
) || isa
<ResumeInst
>(Inst
) ||
1127 isa
<CleanupReturnInst
>(Inst
))
1128 RetVec
.push_back(&Inst
);
1130 if (auto *DDI
= dyn_cast
<DbgDeclareInst
>(&Inst
))
1131 if (auto *Alloca
= dyn_cast_or_null
<AllocaInst
>(DDI
->getAddress()))
1132 AllocaDeclareMap
[Alloca
].push_back(DDI
);
1134 if (InstrumentLandingPads
&& isa
<LandingPadInst
>(Inst
))
1135 LandingPadVec
.push_back(&Inst
);
1137 Value
*MaybeMask
= nullptr;
1141 Value
*Addr
= isInterestingMemoryAccess(&Inst
, &IsWrite
, &TypeSize
,
1142 &Alignment
, &MaybeMask
);
1143 if (Addr
|| isa
<MemIntrinsic
>(Inst
))
1144 ToInstrument
.push_back(&Inst
);
1148 initializeCallbacks(*F
.getParent());
1150 if (!LandingPadVec
.empty())
1151 instrumentLandingPads(LandingPadVec
);
1153 if (AllocasToInstrument
.empty() && F
.hasPersonalityFn() &&
1154 F
.getPersonalityFn()->getName() == kHwasanPersonalityThunkName
) {
1155 // __hwasan_personality_thunk is a no-op for functions without an
1156 // instrumented stack, so we can drop it.
1157 F
.setPersonalityFn(nullptr);
1160 if (AllocasToInstrument
.empty() && ToInstrument
.empty())
1163 assert(!LocalDynamicShadow
);
1165 Instruction
*InsertPt
= &*F
.getEntryBlock().begin();
1166 IRBuilder
<> EntryIRB(InsertPt
);
1167 emitPrologue(EntryIRB
,
1168 /*WithFrameRecord*/ ClRecordStackHistory
&&
1169 !AllocasToInstrument
.empty());
1171 bool Changed
= false;
1172 if (!AllocasToInstrument
.empty()) {
1174 ClGenerateTagsWithCalls
? nullptr : getStackBaseTag(EntryIRB
);
1175 Changed
|= instrumentStack(AllocasToInstrument
, AllocaDeclareMap
, RetVec
,
1179 // Pad and align each of the allocas that we instrumented to stop small
1180 // uninteresting allocas from hiding in instrumented alloca's padding and so
1181 // that we have enough space to store real tags for short granules.
1182 DenseMap
<AllocaInst
*, AllocaInst
*> AllocaToPaddedAllocaMap
;
1183 for (AllocaInst
*AI
: AllocasToInstrument
) {
1184 uint64_t Size
= getAllocaSizeInBytes(*AI
);
1185 uint64_t AlignedSize
= alignTo(Size
, Mapping
.getObjectAlignment());
1187 MaybeAlign(std::max(AI
->getAlignment(), Mapping
.getObjectAlignment())));
1188 if (Size
!= AlignedSize
) {
1189 Type
*AllocatedType
= AI
->getAllocatedType();
1190 if (AI
->isArrayAllocation()) {
1191 uint64_t ArraySize
=
1192 cast
<ConstantInt
>(AI
->getArraySize())->getZExtValue();
1193 AllocatedType
= ArrayType::get(AllocatedType
, ArraySize
);
1195 Type
*TypeWithPadding
= StructType::get(
1196 AllocatedType
, ArrayType::get(Int8Ty
, AlignedSize
- Size
));
1197 auto *NewAI
= new AllocaInst(
1198 TypeWithPadding
, AI
->getType()->getAddressSpace(), nullptr, "", AI
);
1199 NewAI
->takeName(AI
);
1200 NewAI
->setAlignment(MaybeAlign(AI
->getAlignment()));
1201 NewAI
->setUsedWithInAlloca(AI
->isUsedWithInAlloca());
1202 NewAI
->setSwiftError(AI
->isSwiftError());
1203 NewAI
->copyMetadata(*AI
);
1204 auto *Bitcast
= new BitCastInst(NewAI
, AI
->getType(), "", AI
);
1205 AI
->replaceAllUsesWith(Bitcast
);
1206 AllocaToPaddedAllocaMap
[AI
] = NewAI
;
1210 if (!AllocaToPaddedAllocaMap
.empty()) {
1212 for (auto &Inst
: BB
)
1213 if (auto *DVI
= dyn_cast
<DbgVariableIntrinsic
>(&Inst
))
1215 dyn_cast_or_null
<AllocaInst
>(DVI
->getVariableLocation()))
1216 if (auto *NewAI
= AllocaToPaddedAllocaMap
.lookup(AI
))
1218 0, MetadataAsValue::get(*C
, LocalAsMetadata::get(NewAI
)));
1219 for (auto &P
: AllocaToPaddedAllocaMap
)
1220 P
.first
->eraseFromParent();
1223 // If we split the entry block, move any allocas that were originally in the
1224 // entry block back into the entry block so that they aren't treated as
1226 if (EntryIRB
.GetInsertBlock() != &F
.getEntryBlock()) {
1227 InsertPt
= &*F
.getEntryBlock().begin();
1228 for (auto II
= EntryIRB
.GetInsertBlock()->begin(),
1229 IE
= EntryIRB
.GetInsertBlock()->end();
1231 Instruction
*I
= &*II
++;
1232 if (auto *AI
= dyn_cast
<AllocaInst
>(I
))
1233 if (isa
<ConstantInt
>(AI
->getArraySize()))
1234 I
->moveBefore(InsertPt
);
1238 for (auto Inst
: ToInstrument
)
1239 Changed
|= instrumentMemAccess(Inst
);
1241 LocalDynamicShadow
= nullptr;
1242 StackBaseTag
= nullptr;
1247 void HWAddressSanitizer::instrumentGlobal(GlobalVariable
*GV
, uint8_t Tag
) {
1248 Constant
*Initializer
= GV
->getInitializer();
1249 uint64_t SizeInBytes
=
1250 M
.getDataLayout().getTypeAllocSize(Initializer
->getType());
1251 uint64_t NewSize
= alignTo(SizeInBytes
, Mapping
.getObjectAlignment());
1252 if (SizeInBytes
!= NewSize
) {
1253 // Pad the initializer out to the next multiple of 16 bytes and add the
1254 // required short granule tag.
1255 std::vector
<uint8_t> Init(NewSize
- SizeInBytes
, 0);
1257 Constant
*Padding
= ConstantDataArray::get(*C
, Init
);
1258 Initializer
= ConstantStruct::getAnon({Initializer
, Padding
});
1261 auto *NewGV
= new GlobalVariable(M
, Initializer
->getType(), GV
->isConstant(),
1262 GlobalValue::ExternalLinkage
, Initializer
,
1263 GV
->getName() + ".hwasan");
1264 NewGV
->copyAttributesFrom(GV
);
1265 NewGV
->setLinkage(GlobalValue::PrivateLinkage
);
1266 NewGV
->copyMetadata(GV
, 0);
1267 NewGV
->setAlignment(
1268 MaybeAlign(std::max(GV
->getAlignment(), Mapping
.getObjectAlignment())));
1270 // It is invalid to ICF two globals that have different tags. In the case
1271 // where the size of the global is a multiple of the tag granularity the
1272 // contents of the globals may be the same but the tags (i.e. symbol values)
1273 // may be different, and the symbols are not considered during ICF. In the
1274 // case where the size is not a multiple of the granularity, the short granule
1275 // tags would discriminate two globals with different tags, but there would
1276 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1277 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1278 // granule tag in the last byte.
1279 NewGV
->setUnnamedAddr(GlobalValue::UnnamedAddr::None
);
1281 // Descriptor format (assuming little-endian):
1282 // bytes 0-3: relative address of global
1283 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1284 // it isn't, we create multiple descriptors)
1286 auto *DescriptorTy
= StructType::get(Int32Ty
, Int32Ty
);
1287 const uint64_t MaxDescriptorSize
= 0xfffff0;
1288 for (uint64_t DescriptorPos
= 0; DescriptorPos
< SizeInBytes
;
1289 DescriptorPos
+= MaxDescriptorSize
) {
1291 new GlobalVariable(M
, DescriptorTy
, true, GlobalValue::PrivateLinkage
,
1292 nullptr, GV
->getName() + ".hwasan.descriptor");
1293 auto *GVRelPtr
= ConstantExpr::getTrunc(
1294 ConstantExpr::getAdd(
1295 ConstantExpr::getSub(
1296 ConstantExpr::getPtrToInt(NewGV
, Int64Ty
),
1297 ConstantExpr::getPtrToInt(Descriptor
, Int64Ty
)),
1298 ConstantInt::get(Int64Ty
, DescriptorPos
)),
1300 uint32_t Size
= std::min(SizeInBytes
- DescriptorPos
, MaxDescriptorSize
);
1301 auto *SizeAndTag
= ConstantInt::get(Int32Ty
, Size
| (uint32_t(Tag
) << 24));
1302 Descriptor
->setComdat(NewGV
->getComdat());
1303 Descriptor
->setInitializer(ConstantStruct::getAnon({GVRelPtr
, SizeAndTag
}));
1304 Descriptor
->setSection("hwasan_globals");
1305 Descriptor
->setMetadata(LLVMContext::MD_associated
,
1306 MDNode::get(*C
, ValueAsMetadata::get(NewGV
)));
1307 appendToCompilerUsed(M
, Descriptor
);
1310 Constant
*Aliasee
= ConstantExpr::getIntToPtr(
1311 ConstantExpr::getAdd(
1312 ConstantExpr::getPtrToInt(NewGV
, Int64Ty
),
1313 ConstantInt::get(Int64Ty
, uint64_t(Tag
) << kPointerTagShift
)),
1315 auto *Alias
= GlobalAlias::create(GV
->getValueType(), GV
->getAddressSpace(),
1316 GV
->getLinkage(), "", Aliasee
, &M
);
1317 Alias
->setVisibility(GV
->getVisibility());
1318 Alias
->takeName(GV
);
1319 GV
->replaceAllUsesWith(Alias
);
1320 GV
->eraseFromParent();
1323 void HWAddressSanitizer::instrumentGlobals() {
1324 // Start by creating a note that contains pointers to the list of global
1325 // descriptors. Adding a note to the output file will cause the linker to
1326 // create a PT_NOTE program header pointing to the note that we can use to
1327 // find the descriptor list starting from the program headers. A function
1328 // provided by the runtime initializes the shadow memory for the globals by
1329 // accessing the descriptor list via the note. The dynamic loader needs to
1330 // call this function whenever a library is loaded.
1332 // The reason why we use a note for this instead of a more conventional
1333 // approach of having a global constructor pass a descriptor list pointer to
1334 // the runtime is because of an order of initialization problem. With
1335 // constructors we can encounter the following problematic scenario:
1337 // 1) library A depends on library B and also interposes one of B's symbols
1338 // 2) B's constructors are called before A's (as required for correctness)
1339 // 3) during construction, B accesses one of its "own" globals (actually
1340 // interposed by A) and triggers a HWASAN failure due to the initialization
1341 // for A not having happened yet
1343 // Even without interposition it is possible to run into similar situations in
1344 // cases where two libraries mutually depend on each other.
1346 // We only need one note per binary, so put everything for the note in a
1348 Comdat
*NoteComdat
= M
.getOrInsertComdat(kHwasanNoteName
);
1350 Type
*Int8Arr0Ty
= ArrayType::get(Int8Ty
, 0);
1352 new GlobalVariable(M
, Int8Arr0Ty
, true, GlobalVariable::ExternalLinkage
,
1353 nullptr, "__start_hwasan_globals");
1354 Start
->setVisibility(GlobalValue::HiddenVisibility
);
1355 Start
->setDSOLocal(true);
1357 new GlobalVariable(M
, Int8Arr0Ty
, true, GlobalVariable::ExternalLinkage
,
1358 nullptr, "__stop_hwasan_globals");
1359 Stop
->setVisibility(GlobalValue::HiddenVisibility
);
1360 Stop
->setDSOLocal(true);
1362 // Null-terminated so actually 8 bytes, which are required in order to align
1363 // the note properly.
1364 auto *Name
= ConstantDataArray::get(*C
, "LLVM\0\0\0");
1366 auto *NoteTy
= StructType::get(Int32Ty
, Int32Ty
, Int32Ty
, Name
->getType(),
1369 new GlobalVariable(M
, NoteTy
, /*isConstantGlobal=*/true,
1370 GlobalValue::PrivateLinkage
, nullptr, kHwasanNoteName
);
1371 Note
->setSection(".note.hwasan.globals");
1372 Note
->setComdat(NoteComdat
);
1373 Note
->setAlignment(Align(4));
1374 Note
->setDSOLocal(true);
1376 // The pointers in the note need to be relative so that the note ends up being
1377 // placed in rodata, which is the standard location for notes.
1378 auto CreateRelPtr
= [&](Constant
*Ptr
) {
1379 return ConstantExpr::getTrunc(
1380 ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr
, Int64Ty
),
1381 ConstantExpr::getPtrToInt(Note
, Int64Ty
)),
1384 Note
->setInitializer(ConstantStruct::getAnon(
1385 {ConstantInt::get(Int32Ty
, 8), // n_namesz
1386 ConstantInt::get(Int32Ty
, 8), // n_descsz
1387 ConstantInt::get(Int32Ty
, ELF::NT_LLVM_HWASAN_GLOBALS
), // n_type
1388 Name
, CreateRelPtr(Start
), CreateRelPtr(Stop
)}));
1389 appendToCompilerUsed(M
, Note
);
1391 // Create a zero-length global in hwasan_globals so that the linker will
1392 // always create start and stop symbols.
1393 auto Dummy
= new GlobalVariable(
1394 M
, Int8Arr0Ty
, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage
,
1395 Constant::getNullValue(Int8Arr0Ty
), "hwasan.dummy.global");
1396 Dummy
->setSection("hwasan_globals");
1397 Dummy
->setComdat(NoteComdat
);
1398 Dummy
->setMetadata(LLVMContext::MD_associated
,
1399 MDNode::get(*C
, ValueAsMetadata::get(Note
)));
1400 appendToCompilerUsed(M
, Dummy
);
1402 std::vector
<GlobalVariable
*> Globals
;
1403 for (GlobalVariable
&GV
: M
.globals()) {
1404 if (GV
.isDeclarationForLinker() || GV
.getName().startswith("llvm.") ||
1408 // Common symbols can't have aliases point to them, so they can't be tagged.
1409 if (GV
.hasCommonLinkage())
1412 // Globals with custom sections may be used in __start_/__stop_ enumeration,
1413 // which would be broken both by adding tags and potentially by the extra
1414 // padding/alignment that we insert.
1415 if (GV
.hasSection())
1418 Globals
.push_back(&GV
);
1422 Hasher
.update(M
.getSourceFileName());
1423 MD5::MD5Result Hash
;
1425 uint8_t Tag
= Hash
[0];
1427 for (GlobalVariable
*GV
: Globals
) {
1428 // Skip tag 0 in order to avoid collisions with untagged memory.
1431 instrumentGlobal(GV
, Tag
++);
1435 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1436 // We need to untag stack frames as we unwind past them. That is the job of
1437 // the personality function wrapper, which either wraps an existing
1438 // personality function or acts as a personality function on its own. Each
1439 // function that has a personality function or that can be unwound past has
1440 // its personality function changed to a thunk that calls the personality
1441 // function wrapper in the runtime.
1442 MapVector
<Constant
*, std::vector
<Function
*>> PersonalityFns
;
1443 for (Function
&F
: M
) {
1444 if (F
.isDeclaration() || !F
.hasFnAttribute(Attribute::SanitizeHWAddress
))
1447 if (F
.hasPersonalityFn()) {
1448 PersonalityFns
[F
.getPersonalityFn()->stripPointerCasts()].push_back(&F
);
1449 } else if (!F
.hasFnAttribute(Attribute::NoUnwind
)) {
1450 PersonalityFns
[nullptr].push_back(&F
);
1454 if (PersonalityFns
.empty())
1457 FunctionCallee HwasanPersonalityWrapper
= M
.getOrInsertFunction(
1458 "__hwasan_personality_wrapper", Int32Ty
, Int32Ty
, Int32Ty
, Int64Ty
,
1459 Int8PtrTy
, Int8PtrTy
, Int8PtrTy
, Int8PtrTy
, Int8PtrTy
);
1460 FunctionCallee UnwindGetGR
= M
.getOrInsertFunction("_Unwind_GetGR", VoidTy
);
1461 FunctionCallee UnwindGetCFA
= M
.getOrInsertFunction("_Unwind_GetCFA", VoidTy
);
1463 for (auto &P
: PersonalityFns
) {
1464 std::string ThunkName
= kHwasanPersonalityThunkName
;
1466 ThunkName
+= ("." + P
.first
->getName()).str();
1467 FunctionType
*ThunkFnTy
= FunctionType::get(
1468 Int32Ty
, {Int32Ty
, Int32Ty
, Int64Ty
, Int8PtrTy
, Int8PtrTy
}, false);
1469 bool IsLocal
= P
.first
&& (!isa
<GlobalValue
>(P
.first
) ||
1470 cast
<GlobalValue
>(P
.first
)->hasLocalLinkage());
1471 auto *ThunkFn
= Function::Create(ThunkFnTy
,
1472 IsLocal
? GlobalValue::InternalLinkage
1473 : GlobalValue::LinkOnceODRLinkage
,
1476 ThunkFn
->setVisibility(GlobalValue::HiddenVisibility
);
1477 ThunkFn
->setComdat(M
.getOrInsertComdat(ThunkName
));
1480 auto *BB
= BasicBlock::Create(*C
, "entry", ThunkFn
);
1481 IRBuilder
<> IRB(BB
);
1482 CallInst
*WrapperCall
= IRB
.CreateCall(
1483 HwasanPersonalityWrapper
,
1484 {ThunkFn
->getArg(0), ThunkFn
->getArg(1), ThunkFn
->getArg(2),
1485 ThunkFn
->getArg(3), ThunkFn
->getArg(4),
1486 P
.first
? IRB
.CreateBitCast(P
.first
, Int8PtrTy
)
1487 : Constant::getNullValue(Int8PtrTy
),
1488 IRB
.CreateBitCast(UnwindGetGR
.getCallee(), Int8PtrTy
),
1489 IRB
.CreateBitCast(UnwindGetCFA
.getCallee(), Int8PtrTy
)});
1490 WrapperCall
->setTailCall();
1491 IRB
.CreateRet(WrapperCall
);
1493 for (Function
*F
: P
.second
)
1494 F
->setPersonalityFn(ThunkFn
);
1498 void HWAddressSanitizer::ShadowMapping::init(Triple
&TargetTriple
) {
1499 Scale
= kDefaultShadowScale
;
1500 if (ClMappingOffset
.getNumOccurrences() > 0) {
1503 Offset
= ClMappingOffset
;
1504 } else if (ClEnableKhwasan
|| ClInstrumentWithCalls
) {
1508 } else if (ClWithIfunc
) {
1511 Offset
= kDynamicShadowSentinel
;
1512 } else if (ClWithTls
) {
1515 Offset
= kDynamicShadowSentinel
;
1519 Offset
= kDynamicShadowSentinel
;