1 //===- AArch64StackTagging.cpp - Stack tagging in IR --===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
9 //===----------------------------------------------------------------------===//
12 #include "AArch64InstrInfo.h"
13 #include "AArch64Subtarget.h"
14 #include "AArch64TargetMachine.h"
15 #include "llvm/ADT/DenseMap.h"
16 #include "llvm/ADT/DepthFirstIterator.h"
17 #include "llvm/ADT/MapVector.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/CFG.h"
23 #include "llvm/Analysis/LoopInfo.h"
24 #include "llvm/Analysis/ScalarEvolution.h"
25 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
26 #include "llvm/Analysis/ValueTracking.h"
27 #include "llvm/CodeGen/LiveRegUnits.h"
28 #include "llvm/CodeGen/MachineBasicBlock.h"
29 #include "llvm/CodeGen/MachineFunction.h"
30 #include "llvm/CodeGen/MachineFunctionPass.h"
31 #include "llvm/CodeGen/MachineInstr.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineLoopInfo.h"
34 #include "llvm/CodeGen/MachineOperand.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/TargetPassConfig.h"
37 #include "llvm/CodeGen/TargetRegisterInfo.h"
38 #include "llvm/IR/DebugLoc.h"
39 #include "llvm/IR/Dominators.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/GetElementPtrTypeIterator.h"
42 #include "llvm/IR/Instruction.h"
43 #include "llvm/IR/Instructions.h"
44 #include "llvm/IR/IntrinsicInst.h"
45 #include "llvm/IR/Metadata.h"
46 #include "llvm/Pass.h"
47 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Debug.h"
49 #include "llvm/Support/raw_ostream.h"
50 #include "llvm/Transforms/Utils/Local.h"
57 #define DEBUG_TYPE "stack-tagging"
59 static cl::opt
<bool> ClMergeInit(
60 "stack-tagging-merge-init", cl::Hidden
, cl::init(true), cl::ZeroOrMore
,
61 cl::desc("merge stack variable initializers with tagging when possible"));
63 static cl::opt
<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit",
64 cl::init(40), cl::Hidden
);
66 static const Align kTagGranuleSize
= Align(16);
70 class InitializerBuilder
{
75 Function
*SetTagZeroFn
;
78 // List of initializers sorted by start offset.
83 SmallVector
<Range
, 4> Ranges
;
84 // 8-aligned offset => 8-byte initializer
85 // Missing keys are zero initialized.
86 std::map
<uint64_t, Value
*> Out
;
89 InitializerBuilder(uint64_t Size
, const DataLayout
*DL
, Value
*BasePtr
,
90 Function
*SetTagFn
, Function
*SetTagZeroFn
,
92 : Size(Size
), DL(DL
), BasePtr(BasePtr
), SetTagFn(SetTagFn
),
93 SetTagZeroFn(SetTagZeroFn
), StgpFn(StgpFn
) {}
95 bool addRange(uint64_t Start
, uint64_t End
, Instruction
*Inst
) {
96 auto I
= std::lower_bound(
97 Ranges
.begin(), Ranges
.end(), Start
,
98 [](const Range
&LHS
, uint64_t RHS
) { return LHS
.End
<= RHS
; });
99 if (I
!= Ranges
.end() && End
> I
->Start
) {
103 Ranges
.insert(I
, {Start
, End
, Inst
});
107 bool addStore(uint64_t Offset
, StoreInst
*SI
, const DataLayout
*DL
) {
108 int64_t StoreSize
= DL
->getTypeStoreSize(SI
->getOperand(0)->getType());
109 if (!addRange(Offset
, Offset
+ StoreSize
, SI
))
112 applyStore(IRB
, Offset
, Offset
+ StoreSize
, SI
->getOperand(0));
116 bool addMemSet(uint64_t Offset
, MemSetInst
*MSI
) {
117 uint64_t StoreSize
= cast
<ConstantInt
>(MSI
->getLength())->getZExtValue();
118 if (!addRange(Offset
, Offset
+ StoreSize
, MSI
))
120 IRBuilder
<> IRB(MSI
);
121 applyMemSet(IRB
, Offset
, Offset
+ StoreSize
,
122 cast
<ConstantInt
>(MSI
->getValue()));
126 void applyMemSet(IRBuilder
<> &IRB
, int64_t Start
, int64_t End
,
128 // Out[] does not distinguish between zero and undef, and we already know
129 // that this memset does not overlap with any other initializer. Nothing to
133 for (int64_t Offset
= Start
- Start
% 8; Offset
< End
; Offset
+= 8) {
134 uint64_t Cst
= 0x0101010101010101UL
;
135 int LowBits
= Offset
< Start
? (Start
- Offset
) * 8 : 0;
137 Cst
= (Cst
>> LowBits
) << LowBits
;
138 int HighBits
= End
- Offset
< 8 ? (8 - (End
- Offset
)) * 8 : 0;
140 Cst
= (Cst
<< HighBits
) >> HighBits
;
142 ConstantInt::get(IRB
.getInt64Ty(), Cst
* V
->getZExtValue());
144 Value
*&CurrentV
= Out
[Offset
];
148 CurrentV
= IRB
.CreateOr(CurrentV
, C
);
153 // Take a 64-bit slice of the value starting at the given offset (in bytes).
154 // Offset can be negative. Pad with zeroes on both sides when necessary.
155 Value
*sliceValue(IRBuilder
<> &IRB
, Value
*V
, int64_t Offset
) {
157 V
= IRB
.CreateLShr(V
, Offset
* 8);
158 V
= IRB
.CreateZExtOrTrunc(V
, IRB
.getInt64Ty());
159 } else if (Offset
< 0) {
160 V
= IRB
.CreateZExtOrTrunc(V
, IRB
.getInt64Ty());
161 V
= IRB
.CreateShl(V
, -Offset
* 8);
163 V
= IRB
.CreateZExtOrTrunc(V
, IRB
.getInt64Ty());
168 void applyStore(IRBuilder
<> &IRB
, int64_t Start
, int64_t End
,
169 Value
*StoredValue
) {
170 StoredValue
= flatten(IRB
, StoredValue
);
171 for (int64_t Offset
= Start
- Start
% 8; Offset
< End
; Offset
+= 8) {
172 Value
*V
= sliceValue(IRB
, StoredValue
, Offset
- Start
);
173 Value
*&CurrentV
= Out
[Offset
];
177 CurrentV
= IRB
.CreateOr(CurrentV
, V
);
182 void generate(IRBuilder
<> &IRB
) {
183 LLVM_DEBUG(dbgs() << "Combined initializer\n");
184 // No initializers => the entire allocation is undef.
185 if (Ranges
.empty()) {
186 emitUndef(IRB
, 0, Size
);
190 // Look through 8-byte initializer list 16 bytes at a time;
191 // If one of the two 8-byte halfs is non-zero non-undef, emit STGP.
192 // Otherwise, emit zeroes up to next available item.
193 uint64_t LastOffset
= 0;
194 for (uint64_t Offset
= 0; Offset
< Size
; Offset
+= 16) {
195 auto I1
= Out
.find(Offset
);
196 auto I2
= Out
.find(Offset
+ 8);
197 if (I1
== Out
.end() && I2
== Out
.end())
200 if (Offset
> LastOffset
)
201 emitZeroes(IRB
, LastOffset
, Offset
- LastOffset
);
203 Value
*Store1
= I1
== Out
.end() ? Constant::getNullValue(IRB
.getInt64Ty())
205 Value
*Store2
= I2
== Out
.end() ? Constant::getNullValue(IRB
.getInt64Ty())
207 emitPair(IRB
, Offset
, Store1
, Store2
);
208 LastOffset
= Offset
+ 16;
211 // memset(0) does not update Out[], therefore the tail can be either undef
213 if (LastOffset
< Size
)
214 emitZeroes(IRB
, LastOffset
, Size
- LastOffset
);
216 for (const auto &R
: Ranges
) {
217 R
.Inst
->eraseFromParent();
221 void emitZeroes(IRBuilder
<> &IRB
, uint64_t Offset
, uint64_t Size
) {
222 LLVM_DEBUG(dbgs() << " [" << Offset
<< ", " << Offset
+ Size
224 Value
*Ptr
= BasePtr
;
226 Ptr
= IRB
.CreateConstGEP1_32(Ptr
, Offset
);
227 IRB
.CreateCall(SetTagZeroFn
,
228 {Ptr
, ConstantInt::get(IRB
.getInt64Ty(), Size
)});
231 void emitUndef(IRBuilder
<> &IRB
, uint64_t Offset
, uint64_t Size
) {
232 LLVM_DEBUG(dbgs() << " [" << Offset
<< ", " << Offset
+ Size
234 Value
*Ptr
= BasePtr
;
236 Ptr
= IRB
.CreateConstGEP1_32(Ptr
, Offset
);
237 IRB
.CreateCall(SetTagFn
, {Ptr
, ConstantInt::get(IRB
.getInt64Ty(), Size
)});
240 void emitPair(IRBuilder
<> &IRB
, uint64_t Offset
, Value
*A
, Value
*B
) {
241 LLVM_DEBUG(dbgs() << " [" << Offset
<< ", " << Offset
+ 16 << "):\n");
242 LLVM_DEBUG(dbgs() << " " << *A
<< "\n " << *B
<< "\n");
243 Value
*Ptr
= BasePtr
;
245 Ptr
= IRB
.CreateConstGEP1_32(Ptr
, Offset
);
246 IRB
.CreateCall(StgpFn
, {Ptr
, A
, B
});
249 Value
*flatten(IRBuilder
<> &IRB
, Value
*V
) {
250 if (V
->getType()->isIntegerTy())
252 // vector of pointers -> vector of ints
253 if (VectorType
*VecTy
= dyn_cast
<VectorType
>(V
->getType())) {
254 LLVMContext
&Ctx
= IRB
.getContext();
255 Type
*EltTy
= VecTy
->getElementType();
256 if (EltTy
->isPointerTy()) {
257 uint32_t EltSize
= DL
->getTypeSizeInBits(EltTy
);
258 Type
*NewTy
= VectorType::get(IntegerType::get(Ctx
, EltSize
),
259 VecTy
->getNumElements());
260 V
= IRB
.CreatePointerCast(V
, NewTy
);
263 return IRB
.CreateBitOrPointerCast(
264 V
, IRB
.getIntNTy(DL
->getTypeStoreSize(V
->getType()) * 8));
268 class AArch64StackTagging
: public FunctionPass
{
271 SmallVector
<IntrinsicInst
*, 2> LifetimeStart
;
272 SmallVector
<IntrinsicInst
*, 2> LifetimeEnd
;
273 SmallVector
<DbgVariableIntrinsic
*, 2> DbgVariableIntrinsics
;
274 int Tag
; // -1 for non-tagged allocations
280 static char ID
; // Pass ID, replacement for typeid
282 AArch64StackTagging(bool MergeInit
= true)
284 MergeInit(ClMergeInit
.getNumOccurrences() > 0 ? ClMergeInit
286 initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry());
289 bool isInterestingAlloca(const AllocaInst
&AI
);
290 void alignAndPadAlloca(AllocaInfo
&Info
);
292 void tagAlloca(AllocaInst
*AI
, Instruction
*InsertBefore
, Value
*Ptr
,
294 void untagAlloca(AllocaInst
*AI
, Instruction
*InsertBefore
, uint64_t Size
);
296 Instruction
*collectInitializers(Instruction
*StartInst
, Value
*StartPtr
,
297 uint64_t Size
, InitializerBuilder
&IB
);
300 insertBaseTaggedPointer(const MapVector
<AllocaInst
*, AllocaInfo
> &Allocas
,
301 const DominatorTree
*DT
);
302 bool runOnFunction(Function
&F
) override
;
304 StringRef
getPassName() const override
{ return "AArch64 Stack Tagging"; }
308 Function
*SetTagFunc
;
309 const DataLayout
*DL
;
312 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
313 AU
.setPreservesCFG();
315 AU
.addRequired
<AAResultsWrapperPass
>();
319 } // end anonymous namespace
321 char AArch64StackTagging::ID
= 0;
323 INITIALIZE_PASS_BEGIN(AArch64StackTagging
, DEBUG_TYPE
, "AArch64 Stack Tagging",
325 INITIALIZE_PASS_END(AArch64StackTagging
, DEBUG_TYPE
, "AArch64 Stack Tagging",
328 FunctionPass
*llvm::createAArch64StackTaggingPass(bool MergeInit
) {
329 return new AArch64StackTagging(MergeInit
);
332 Instruction
*AArch64StackTagging::collectInitializers(Instruction
*StartInst
,
335 InitializerBuilder
&IB
) {
336 MemoryLocation AllocaLoc
{StartPtr
, Size
};
337 Instruction
*LastInst
= StartInst
;
338 BasicBlock::iterator
BI(StartInst
);
341 for (; Count
< ClScanLimit
&& !BI
->isTerminator(); ++BI
) {
342 if (!isa
<DbgInfoIntrinsic
>(*BI
))
345 if (isNoModRef(AA
->getModRefInfo(&*BI
, AllocaLoc
)))
348 if (!isa
<StoreInst
>(BI
) && !isa
<MemSetInst
>(BI
)) {
349 // If the instruction is readnone, ignore it, otherwise bail out. We
350 // don't even allow readonly here because we don't want something like:
351 // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
352 if (BI
->mayWriteToMemory() || BI
->mayReadFromMemory())
357 if (StoreInst
*NextStore
= dyn_cast
<StoreInst
>(BI
)) {
358 if (!NextStore
->isSimple())
361 // Check to see if this store is to a constant offset from the start ptr.
362 Optional
<int64_t> Offset
=
363 isPointerOffset(StartPtr
, NextStore
->getPointerOperand(), *DL
);
367 if (!IB
.addStore(*Offset
, NextStore
, DL
))
369 LastInst
= NextStore
;
371 MemSetInst
*MSI
= cast
<MemSetInst
>(BI
);
373 if (MSI
->isVolatile() || !isa
<ConstantInt
>(MSI
->getLength()))
376 if (!isa
<ConstantInt
>(MSI
->getValue()))
379 // Check to see if this store is to a constant offset from the start ptr.
380 Optional
<int64_t> Offset
= isPointerOffset(StartPtr
, MSI
->getDest(), *DL
);
384 if (!IB
.addMemSet(*Offset
, MSI
))
392 bool AArch64StackTagging::isInterestingAlloca(const AllocaInst
&AI
) {
393 // FIXME: support dynamic allocas
395 AI
.getAllocatedType()->isSized() && AI
.isStaticAlloca() &&
396 // alloca() may be called with 0 size, ignore it.
397 AI
.getAllocationSizeInBits(*DL
).getValue() > 0 &&
398 // inalloca allocas are not treated as static, and we don't want
399 // dynamic alloca instrumentation for them as well.
400 !AI
.isUsedWithInAlloca() &&
401 // swifterror allocas are register promoted by ISel
403 return IsInteresting
;
406 void AArch64StackTagging::tagAlloca(AllocaInst
*AI
, Instruction
*InsertBefore
,
407 Value
*Ptr
, uint64_t Size
) {
408 auto SetTagZeroFunc
=
409 Intrinsic::getDeclaration(F
->getParent(), Intrinsic::aarch64_settag_zero
);
411 Intrinsic::getDeclaration(F
->getParent(), Intrinsic::aarch64_stgp
);
413 InitializerBuilder
IB(Size
, DL
, Ptr
, SetTagFunc
, SetTagZeroFunc
, StgpFunc
);
415 Triple(AI
->getModule()->getTargetTriple()).isLittleEndian();
416 // Current implementation of initializer merging assumes little endianness.
417 if (MergeInit
&& !F
->hasOptNone() && LittleEndian
) {
418 LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI
419 << ", size = " << Size
<< "\n");
420 InsertBefore
= collectInitializers(InsertBefore
, Ptr
, Size
, IB
);
423 IRBuilder
<> IRB(InsertBefore
);
427 void AArch64StackTagging::untagAlloca(AllocaInst
*AI
, Instruction
*InsertBefore
,
429 IRBuilder
<> IRB(InsertBefore
);
430 IRB
.CreateCall(SetTagFunc
, {IRB
.CreatePointerCast(AI
, IRB
.getInt8PtrTy()),
431 ConstantInt::get(IRB
.getInt64Ty(), Size
)});
434 Instruction
*AArch64StackTagging::insertBaseTaggedPointer(
435 const MapVector
<AllocaInst
*, AllocaInfo
> &Allocas
,
436 const DominatorTree
*DT
) {
437 BasicBlock
*PrologueBB
= nullptr;
438 // Try sinking IRG as deep as possible to avoid hurting shrink wrap.
439 for (auto &I
: Allocas
) {
440 const AllocaInfo
&Info
= I
.second
;
441 AllocaInst
*AI
= Info
.AI
;
445 PrologueBB
= AI
->getParent();
448 PrologueBB
= DT
->findNearestCommonDominator(PrologueBB
, AI
->getParent());
452 IRBuilder
<> IRB(&PrologueBB
->front());
454 Intrinsic::getDeclaration(F
->getParent(), Intrinsic::aarch64_irg_sp
);
456 IRB
.CreateCall(IRG_SP
, {Constant::getNullValue(IRB
.getInt64Ty())});
457 Base
->setName("basetag");
461 void AArch64StackTagging::alignAndPadAlloca(AllocaInfo
&Info
) {
462 const Align NewAlignment
=
463 max(MaybeAlign(Info
.AI
->getAlignment()), kTagGranuleSize
);
464 Info
.AI
->setAlignment(NewAlignment
);
466 uint64_t Size
= Info
.AI
->getAllocationSizeInBits(*DL
).getValue() / 8;
467 uint64_t AlignedSize
= alignTo(Size
, kTagGranuleSize
);
468 if (Size
== AlignedSize
)
471 // Add padding to the alloca.
472 Type
*AllocatedType
=
473 Info
.AI
->isArrayAllocation()
475 Info
.AI
->getAllocatedType(),
476 cast
<ConstantInt
>(Info
.AI
->getArraySize())->getZExtValue())
477 : Info
.AI
->getAllocatedType();
479 ArrayType::get(Type::getInt8Ty(F
->getContext()), AlignedSize
- Size
);
480 Type
*TypeWithPadding
= StructType::get(AllocatedType
, PaddingType
);
481 auto *NewAI
= new AllocaInst(
482 TypeWithPadding
, Info
.AI
->getType()->getAddressSpace(), nullptr, "", Info
.AI
);
483 NewAI
->takeName(Info
.AI
);
484 NewAI
->setAlignment(MaybeAlign(Info
.AI
->getAlignment()));
485 NewAI
->setUsedWithInAlloca(Info
.AI
->isUsedWithInAlloca());
486 NewAI
->setSwiftError(Info
.AI
->isSwiftError());
487 NewAI
->copyMetadata(*Info
.AI
);
489 auto *NewPtr
= new BitCastInst(NewAI
, Info
.AI
->getType(), "", Info
.AI
);
490 Info
.AI
->replaceAllUsesWith(NewPtr
);
491 Info
.AI
->eraseFromParent();
495 // Helper function to check for post-dominance.
496 static bool postDominates(const PostDominatorTree
*PDT
, const IntrinsicInst
*A
,
497 const IntrinsicInst
*B
) {
498 const BasicBlock
*ABB
= A
->getParent();
499 const BasicBlock
*BBB
= B
->getParent();
502 return PDT
->dominates(ABB
, BBB
);
504 for (const Instruction
&I
: *ABB
) {
510 llvm_unreachable("Corrupt instruction list");
513 // FIXME: check for MTE extension
514 bool AArch64StackTagging::runOnFunction(Function
&Fn
) {
515 if (!Fn
.hasFnAttribute(Attribute::SanitizeMemTag
))
519 DL
= &Fn
.getParent()->getDataLayout();
521 AA
= &getAnalysis
<AAResultsWrapperPass
>().getAAResults();
523 MapVector
<AllocaInst
*, AllocaInfo
> Allocas
; // need stable iteration order
524 SmallVector
<Instruction
*, 8> RetVec
;
525 DenseMap
<Value
*, AllocaInst
*> AllocaForValue
;
526 SmallVector
<Instruction
*, 4> UnrecognizedLifetimes
;
528 for (auto &BB
: *F
) {
529 for (BasicBlock::iterator IT
= BB
.begin(); IT
!= BB
.end(); ++IT
) {
530 Instruction
*I
= &*IT
;
531 if (auto *AI
= dyn_cast
<AllocaInst
>(I
)) {
536 if (auto *DVI
= dyn_cast
<DbgVariableIntrinsic
>(I
)) {
538 dyn_cast_or_null
<AllocaInst
>(DVI
->getVariableLocation())) {
539 Allocas
[AI
].DbgVariableIntrinsics
.push_back(DVI
);
544 auto *II
= dyn_cast
<IntrinsicInst
>(I
);
545 if (II
&& (II
->getIntrinsicID() == Intrinsic::lifetime_start
||
546 II
->getIntrinsicID() == Intrinsic::lifetime_end
)) {
548 llvm::findAllocaForValue(II
->getArgOperand(1), AllocaForValue
);
550 UnrecognizedLifetimes
.push_back(I
);
553 if (II
->getIntrinsicID() == Intrinsic::lifetime_start
)
554 Allocas
[AI
].LifetimeStart
.push_back(II
);
556 Allocas
[AI
].LifetimeEnd
.push_back(II
);
559 if (isa
<ReturnInst
>(I
) || isa
<ResumeInst
>(I
) || isa
<CleanupReturnInst
>(I
))
568 int NumInterestingAllocas
= 0;
569 for (auto &I
: Allocas
) {
570 AllocaInfo
&Info
= I
.second
;
573 if (!isInterestingAlloca(*Info
.AI
)) {
578 alignAndPadAlloca(Info
);
579 NumInterestingAllocas
++;
581 NextTag
= (NextTag
+ 1) % 16;
584 if (NumInterestingAllocas
== 0)
587 std::unique_ptr
<DominatorTree
> DeleteDT
;
588 DominatorTree
*DT
= nullptr;
589 if (auto *P
= getAnalysisIfAvailable
<DominatorTreeWrapperPass
>())
590 DT
= &P
->getDomTree();
592 if (DT
== nullptr && (NumInterestingAllocas
> 1 ||
593 !F
->hasFnAttribute(Attribute::OptimizeNone
))) {
594 DeleteDT
= std::make_unique
<DominatorTree
>(*F
);
598 std::unique_ptr
<PostDominatorTree
> DeletePDT
;
599 PostDominatorTree
*PDT
= nullptr;
600 if (auto *P
= getAnalysisIfAvailable
<PostDominatorTreeWrapperPass
>())
601 PDT
= &P
->getPostDomTree();
603 if (PDT
== nullptr && !F
->hasFnAttribute(Attribute::OptimizeNone
)) {
604 DeletePDT
= std::make_unique
<PostDominatorTree
>(*F
);
605 PDT
= DeletePDT
.get();
609 Intrinsic::getDeclaration(F
->getParent(), Intrinsic::aarch64_settag
);
611 Instruction
*Base
= insertBaseTaggedPointer(Allocas
, DT
);
613 for (auto &I
: Allocas
) {
614 const AllocaInfo
&Info
= I
.second
;
615 AllocaInst
*AI
= Info
.AI
;
619 // Replace alloca with tagp(alloca).
620 IRBuilder
<> IRB(Info
.AI
->getNextNode());
621 Function
*TagP
= Intrinsic::getDeclaration(
622 F
->getParent(), Intrinsic::aarch64_tagp
, {Info
.AI
->getType()});
623 Instruction
*TagPCall
=
624 IRB
.CreateCall(TagP
, {Constant::getNullValue(Info
.AI
->getType()), Base
,
625 ConstantInt::get(IRB
.getInt64Ty(), Info
.Tag
)});
626 if (Info
.AI
->hasName())
627 TagPCall
->setName(Info
.AI
->getName() + ".tag");
628 Info
.AI
->replaceAllUsesWith(TagPCall
);
629 TagPCall
->setOperand(0, Info
.AI
);
631 if (UnrecognizedLifetimes
.empty() && Info
.LifetimeStart
.size() == 1 &&
632 Info
.LifetimeEnd
.size() == 1) {
633 IntrinsicInst
*Start
= Info
.LifetimeStart
[0];
634 IntrinsicInst
*End
= Info
.LifetimeEnd
[0];
636 dyn_cast
<ConstantInt
>(Start
->getArgOperand(0))->getZExtValue();
637 Size
= alignTo(Size
, kTagGranuleSize
);
638 tagAlloca(AI
, Start
->getNextNode(), Start
->getArgOperand(1), Size
);
639 // We need to ensure that if we tag some object, we certainly untag it
640 // before the function exits.
641 if (PDT
!= nullptr && postDominates(PDT
, End
, Start
)) {
642 untagAlloca(AI
, End
, Size
);
644 SmallVector
<Instruction
*, 8> ReachableRetVec
;
645 unsigned NumCoveredExits
= 0;
646 for (auto &RI
: RetVec
) {
647 if (!isPotentiallyReachable(Start
, RI
, nullptr, DT
))
649 ReachableRetVec
.push_back(RI
);
650 if (DT
!= nullptr && DT
->dominates(End
, RI
))
653 // If there's a mix of covered and non-covered exits, just put the untag
654 // on exits, so we avoid the redundancy of untagging twice.
655 if (NumCoveredExits
== ReachableRetVec
.size()) {
656 untagAlloca(AI
, End
, Size
);
658 for (auto &RI
: ReachableRetVec
)
659 untagAlloca(AI
, RI
, Size
);
660 // We may have inserted untag outside of the lifetime interval.
661 // Remove the lifetime end call for this alloca.
662 End
->eraseFromParent();
666 uint64_t Size
= Info
.AI
->getAllocationSizeInBits(*DL
).getValue() / 8;
667 Value
*Ptr
= IRB
.CreatePointerCast(TagPCall
, IRB
.getInt8PtrTy());
668 tagAlloca(AI
, &*IRB
.GetInsertPoint(), Ptr
, Size
);
669 for (auto &RI
: RetVec
) {
670 untagAlloca(AI
, RI
, Size
);
672 // We may have inserted tag/untag outside of any lifetime interval.
673 // Remove all lifetime intrinsics for this alloca.
674 for (auto &II
: Info
.LifetimeStart
)
675 II
->eraseFromParent();
676 for (auto &II
: Info
.LifetimeEnd
)
677 II
->eraseFromParent();
680 // Fixup debug intrinsics to point to the new alloca.
681 for (auto DVI
: Info
.DbgVariableIntrinsics
)
684 MetadataAsValue::get(F
->getContext(), LocalAsMetadata::get(Info
.AI
)));
687 // If we have instrumented at least one alloca, all unrecognized lifetime
688 // instrinsics have to go.
689 for (auto &I
: UnrecognizedLifetimes
)
690 I
->eraseFromParent();